85 static int ena_com_admin_init_sq(struct ena_com_admin_queue *admin_queue)  in ena_com_admin_init_sq()  argument
87 	struct ena_com_dev *ena_dev = admin_queue->ena_dev;  in ena_com_admin_init_sq()
88 	struct ena_com_admin_sq *sq = &admin_queue->sq;  in ena_com_admin_init_sq()
89 	u16 size = ADMIN_SQ_SIZE(admin_queue->q_depth);  in ena_com_admin_init_sq()
91 	sq->entries = dma_alloc_coherent(admin_queue->q_dmadev, size,  in ena_com_admin_init_sq()
108 static int ena_com_admin_init_cq(struct ena_com_admin_queue *admin_queue)  in ena_com_admin_init_cq()  argument
110 	struct ena_com_dev *ena_dev = admin_queue->ena_dev;  in ena_com_admin_init_cq()
111 	struct ena_com_admin_cq *cq = &admin_queue->cq;  in ena_com_admin_init_cq()
112 	u16 size = ADMIN_CQ_SIZE(admin_queue->q_depth);  in ena_com_admin_init_cq()
114 	cq->entries = dma_alloc_coherent(admin_queue->q_dmadev, size,  in ena_com_admin_init_cq()
179 static struct ena_comp_ctx *get_comp_ctxt(struct ena_com_admin_queue *admin_queue,  in get_comp_ctxt()  argument
182 	if (unlikely(command_id >= admin_queue->q_depth)) {  in get_comp_ctxt()
183 		netdev_err(admin_queue->ena_dev->net_device,  in get_comp_ctxt()
185 			   command_id, admin_queue->q_depth);  in get_comp_ctxt()
189 	if (unlikely(!admin_queue->comp_ctx)) {  in get_comp_ctxt()
190 		netdev_err(admin_queue->ena_dev->net_device,  in get_comp_ctxt()
195 	if (unlikely(admin_queue->comp_ctx[command_id].occupied && capture)) {  in get_comp_ctxt()
196 		netdev_err(admin_queue->ena_dev->net_device,  in get_comp_ctxt()
202 		atomic_inc(&admin_queue->outstanding_cmds);  in get_comp_ctxt()
203 		admin_queue->comp_ctx[command_id].occupied = true;  in get_comp_ctxt()
206 	return &admin_queue->comp_ctx[command_id];  in get_comp_ctxt()
209 static struct ena_comp_ctx *__ena_com_submit_admin_cmd(struct ena_com_admin_queue *admin_queue,  in __ena_com_submit_admin_cmd()  argument
220 	queue_size_mask = admin_queue->q_depth - 1;  in __ena_com_submit_admin_cmd()
222 	tail_masked = admin_queue->sq.tail & queue_size_mask;  in __ena_com_submit_admin_cmd()
225 	cnt = (u16)atomic_read(&admin_queue->outstanding_cmds);  in __ena_com_submit_admin_cmd()
226 	if (cnt >= admin_queue->q_depth) {  in __ena_com_submit_admin_cmd()
227 		netdev_dbg(admin_queue->ena_dev->net_device,  in __ena_com_submit_admin_cmd()
229 		admin_queue->stats.out_of_space++;  in __ena_com_submit_admin_cmd()
233 	cmd_id = admin_queue->curr_cmd_id;  in __ena_com_submit_admin_cmd()
235 	cmd->aq_common_descriptor.flags |= admin_queue->sq.phase &  in __ena_com_submit_admin_cmd()
241 	comp_ctx = get_comp_ctxt(admin_queue, cmd_id, true);  in __ena_com_submit_admin_cmd()
252 	memcpy(&admin_queue->sq.entries[tail_masked], cmd, cmd_size_in_bytes);  in __ena_com_submit_admin_cmd()
254 	admin_queue->curr_cmd_id = (admin_queue->curr_cmd_id + 1) &  in __ena_com_submit_admin_cmd()
257 	admin_queue->sq.tail++;  in __ena_com_submit_admin_cmd()
258 	admin_queue->stats.submitted_cmd++;  in __ena_com_submit_admin_cmd()
260 	if (unlikely((admin_queue->sq.tail & queue_size_mask) == 0))  in __ena_com_submit_admin_cmd()
261 		admin_queue->sq.phase = !admin_queue->sq.phase;  in __ena_com_submit_admin_cmd()
263 	writel(admin_queue->sq.tail, admin_queue->sq.db_addr);  in __ena_com_submit_admin_cmd()
268 static int ena_com_init_comp_ctxt(struct ena_com_admin_queue *admin_queue)  in ena_com_init_comp_ctxt()  argument
270 	struct ena_com_dev *ena_dev = admin_queue->ena_dev;  in ena_com_init_comp_ctxt()
271 	size_t size = admin_queue->q_depth * sizeof(struct ena_comp_ctx);  in ena_com_init_comp_ctxt()
275 	admin_queue->comp_ctx =  in ena_com_init_comp_ctxt()
276 		devm_kzalloc(admin_queue->q_dmadev, size, GFP_KERNEL);  in ena_com_init_comp_ctxt()
277 	if (unlikely(!admin_queue->comp_ctx)) {  in ena_com_init_comp_ctxt()
282 	for (i = 0; i < admin_queue->q_depth; i++) {  in ena_com_init_comp_ctxt()
283 		comp_ctx = get_comp_ctxt(admin_queue, i, false);  in ena_com_init_comp_ctxt()
291 static struct ena_comp_ctx *ena_com_submit_admin_cmd(struct ena_com_admin_queue *admin_queue,  in ena_com_submit_admin_cmd()  argument
300 	spin_lock_irqsave(&admin_queue->q_lock, flags);  in ena_com_submit_admin_cmd()
301 	if (unlikely(!admin_queue->running_state)) {  in ena_com_submit_admin_cmd()
302 		spin_unlock_irqrestore(&admin_queue->q_lock, flags);  in ena_com_submit_admin_cmd()
305 	comp_ctx = __ena_com_submit_admin_cmd(admin_queue, cmd,  in ena_com_submit_admin_cmd()
310 		admin_queue->running_state = false;  in ena_com_submit_admin_cmd()
311 	spin_unlock_irqrestore(&admin_queue->q_lock, flags);  in ena_com_submit_admin_cmd()
447 static void ena_com_handle_single_admin_completion(struct ena_com_admin_queue *admin_queue,  in ena_com_handle_single_admin_completion()  argument
456 	comp_ctx = get_comp_ctxt(admin_queue, cmd_id, false);  in ena_com_handle_single_admin_completion()
458 		netdev_err(admin_queue->ena_dev->net_device,  in ena_com_handle_single_admin_completion()
460 		admin_queue->running_state = false;  in ena_com_handle_single_admin_completion()
470 	if (!admin_queue->polling)  in ena_com_handle_single_admin_completion()
474 static void ena_com_handle_admin_completion(struct ena_com_admin_queue *admin_queue)  in ena_com_handle_admin_completion()  argument
481 	head_masked = admin_queue->cq.head & (admin_queue->q_depth - 1);  in ena_com_handle_admin_completion()
482 	phase = admin_queue->cq.phase;  in ena_com_handle_admin_completion()
484 	cqe = &admin_queue->cq.entries[head_masked];  in ena_com_handle_admin_completion()
493 		ena_com_handle_single_admin_completion(admin_queue, cqe);  in ena_com_handle_admin_completion()
497 		if (unlikely(head_masked == admin_queue->q_depth)) {  in ena_com_handle_admin_completion()
502 		cqe = &admin_queue->cq.entries[head_masked];  in ena_com_handle_admin_completion()
505 	admin_queue->cq.head += comp_num;  in ena_com_handle_admin_completion()
506 	admin_queue->cq.phase = phase;  in ena_com_handle_admin_completion()
507 	admin_queue->sq.head += comp_num;  in ena_com_handle_admin_completion()
508 	admin_queue->stats.completed_cmd += comp_num;  in ena_com_handle_admin_completion()
511 static int ena_com_comp_status_to_errno(struct ena_com_admin_queue *admin_queue,  in ena_com_comp_status_to_errno()  argument
515 		netdev_err(admin_queue->ena_dev->net_device,  in ena_com_comp_status_to_errno()
545 						     struct ena_com_admin_queue *admin_queue)  in ena_com_wait_and_process_admin_cq_polling()  argument
552 	timeout = jiffies + usecs_to_jiffies(admin_queue->completion_timeout);  in ena_com_wait_and_process_admin_cq_polling()
555 		spin_lock_irqsave(&admin_queue->q_lock, flags);  in ena_com_wait_and_process_admin_cq_polling()
556 		ena_com_handle_admin_completion(admin_queue);  in ena_com_wait_and_process_admin_cq_polling()
557 		spin_unlock_irqrestore(&admin_queue->q_lock, flags);  in ena_com_wait_and_process_admin_cq_polling()
563 			netdev_err(admin_queue->ena_dev->net_device,  in ena_com_wait_and_process_admin_cq_polling()
566 			spin_lock_irqsave(&admin_queue->q_lock, flags);  in ena_com_wait_and_process_admin_cq_polling()
567 			admin_queue->stats.no_completion++;  in ena_com_wait_and_process_admin_cq_polling()
568 			admin_queue->running_state = false;  in ena_com_wait_and_process_admin_cq_polling()
569 			spin_unlock_irqrestore(&admin_queue->q_lock, flags);  in ena_com_wait_and_process_admin_cq_polling()
576 						 admin_queue->ena_dev->ena_min_poll_delay_us);  in ena_com_wait_and_process_admin_cq_polling()
580 		netdev_err(admin_queue->ena_dev->net_device,  in ena_com_wait_and_process_admin_cq_polling()
582 		spin_lock_irqsave(&admin_queue->q_lock, flags);  in ena_com_wait_and_process_admin_cq_polling()
583 		admin_queue->stats.aborted_cmd++;  in ena_com_wait_and_process_admin_cq_polling()
584 		spin_unlock_irqrestore(&admin_queue->q_lock, flags);  in ena_com_wait_and_process_admin_cq_polling()
592 	ret = ena_com_comp_status_to_errno(admin_queue, comp_ctx->comp_status);  in ena_com_wait_and_process_admin_cq_polling()
594 	comp_ctxt_release(admin_queue, comp_ctx);  in ena_com_wait_and_process_admin_cq_polling()
606 	struct ena_com_admin_queue *admin_queue;  in ena_com_set_llq()  local
613 	admin_queue = &ena_dev->admin_queue;  in ena_com_set_llq()
627 	ret = ena_com_execute_admin_command(admin_queue,  in ena_com_set_llq()
774 							struct ena_com_admin_queue *admin_queue)  in ena_com_wait_and_process_admin_cq_interrupts()  argument
781 					    admin_queue->completion_timeout));  in ena_com_wait_and_process_admin_cq_interrupts()
789 		spin_lock_irqsave(&admin_queue->q_lock, flags);  in ena_com_wait_and_process_admin_cq_interrupts()
790 		ena_com_handle_admin_completion(admin_queue);  in ena_com_wait_and_process_admin_cq_interrupts()
791 		admin_queue->stats.no_completion++;  in ena_com_wait_and_process_admin_cq_interrupts()
792 		spin_unlock_irqrestore(&admin_queue->q_lock, flags);  in ena_com_wait_and_process_admin_cq_interrupts()
795 			netdev_err(admin_queue->ena_dev->net_device,  in ena_com_wait_and_process_admin_cq_interrupts()
798 				   admin_queue->auto_polling ? "ON" : "OFF");  in ena_com_wait_and_process_admin_cq_interrupts()
800 			if (admin_queue->auto_polling)  in ena_com_wait_and_process_admin_cq_interrupts()
801 				admin_queue->polling = true;  in ena_com_wait_and_process_admin_cq_interrupts()
803 			netdev_err(admin_queue->ena_dev->net_device,  in ena_com_wait_and_process_admin_cq_interrupts()
811 		if (!admin_queue->polling) {  in ena_com_wait_and_process_admin_cq_interrupts()
812 			admin_queue->running_state = false;  in ena_com_wait_and_process_admin_cq_interrupts()
818 	ret = ena_com_comp_status_to_errno(admin_queue, comp_ctx->comp_status);  in ena_com_wait_and_process_admin_cq_interrupts()
820 	comp_ctxt_release(admin_queue, comp_ctx);  in ena_com_wait_and_process_admin_cq_interrupts()
894 					     struct ena_com_admin_queue *admin_queue)  in ena_com_wait_and_process_admin_cq()  argument
896 	if (admin_queue->polling)  in ena_com_wait_and_process_admin_cq()
898 								 admin_queue);  in ena_com_wait_and_process_admin_cq()
901 							    admin_queue);  in ena_com_wait_and_process_admin_cq()
907 	struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;  in ena_com_destroy_io_sq()  local
927 	ret = ena_com_execute_admin_command(admin_queue,  in ena_com_destroy_io_sq()
1021 	struct ena_com_admin_queue *admin_queue;  in ena_com_get_feature_ex()  local
1032 	admin_queue = &ena_dev->admin_queue;  in ena_com_get_feature_ex()
1054 	ret = ena_com_execute_admin_command(admin_queue,  in ena_com_get_feature_ex()
1224 	struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;  in ena_com_create_io_sq()  local
1267 	ret = ena_com_execute_admin_command(admin_queue,  in ena_com_create_io_sq()
1351 int ena_com_execute_admin_command(struct ena_com_admin_queue *admin_queue,  in ena_com_execute_admin_command()  argument
1360 	comp_ctx = ena_com_submit_admin_cmd(admin_queue, cmd, cmd_size,  in ena_com_execute_admin_command()
1365 			netdev_dbg(admin_queue->ena_dev->net_device,  in ena_com_execute_admin_command()
1368 			netdev_err(admin_queue->ena_dev->net_device,  in ena_com_execute_admin_command()
1374 	ret = ena_com_wait_and_process_admin_cq(comp_ctx, admin_queue);  in ena_com_execute_admin_command()
1376 		if (admin_queue->running_state)  in ena_com_execute_admin_command()
1377 			netdev_err(admin_queue->ena_dev->net_device,  in ena_com_execute_admin_command()
1380 			netdev_dbg(admin_queue->ena_dev->net_device,  in ena_com_execute_admin_command()
1389 	struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;  in ena_com_create_io_cq()  local
1414 	ret = ena_com_execute_admin_command(admin_queue,  in ena_com_create_io_cq()
1465 	struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;  in ena_com_abort_admin_commands()  local
1469 	if (!admin_queue->comp_ctx)  in ena_com_abort_admin_commands()
1472 	for (i = 0; i < admin_queue->q_depth; i++) {  in ena_com_abort_admin_commands()
1473 		comp_ctx = get_comp_ctxt(admin_queue, i, false);  in ena_com_abort_admin_commands()
1485 	struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;  in ena_com_wait_for_abort_completion()  local
1489 	spin_lock_irqsave(&admin_queue->q_lock, flags);  in ena_com_wait_for_abort_completion()
1490 	while (atomic_read(&admin_queue->outstanding_cmds) != 0) {  in ena_com_wait_for_abort_completion()
1491 		spin_unlock_irqrestore(&admin_queue->q_lock, flags);  in ena_com_wait_for_abort_completion()
1494 		spin_lock_irqsave(&admin_queue->q_lock, flags);  in ena_com_wait_for_abort_completion()
1496 	spin_unlock_irqrestore(&admin_queue->q_lock, flags);  in ena_com_wait_for_abort_completion()
1502 	struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;  in ena_com_destroy_io_cq()  local
1512 	ret = ena_com_execute_admin_command(admin_queue,  in ena_com_destroy_io_cq()
1527 	return ena_dev->admin_queue.running_state;  in ena_com_get_admin_running_state()
1532 	struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;  in ena_com_set_admin_running_state()  local
1535 	spin_lock_irqsave(&admin_queue->q_lock, flags);  in ena_com_set_admin_running_state()
1536 	ena_dev->admin_queue.running_state = state;  in ena_com_set_admin_running_state()
1537 	spin_unlock_irqrestore(&admin_queue->q_lock, flags);  in ena_com_set_admin_running_state()
1554 	struct ena_com_admin_queue *admin_queue;  in ena_com_set_aenq_config()  local
1574 	admin_queue = &ena_dev->admin_queue;  in ena_com_set_aenq_config()
1581 	ret = ena_com_execute_admin_command(admin_queue,  in ena_com_set_aenq_config()
1671 				      struct ena_com_admin_queue *admin_queue)  in ena_com_free_ena_admin_queue_comp_ctx()  argument
1674 	if (!admin_queue->comp_ctx)  in ena_com_free_ena_admin_queue_comp_ctx()
1677 	devm_kfree(ena_dev->dmadev, admin_queue->comp_ctx);  in ena_com_free_ena_admin_queue_comp_ctx()
1679 	admin_queue->comp_ctx = NULL;  in ena_com_free_ena_admin_queue_comp_ctx()
1684 	struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;  in ena_com_admin_destroy()  local
1685 	struct ena_com_admin_cq *cq = &admin_queue->cq;  in ena_com_admin_destroy()
1686 	struct ena_com_admin_sq *sq = &admin_queue->sq;  in ena_com_admin_destroy()
1690 	ena_com_free_ena_admin_queue_comp_ctx(ena_dev, admin_queue);  in ena_com_admin_destroy()
1692 	size = ADMIN_SQ_SIZE(admin_queue->q_depth);  in ena_com_admin_destroy()
1698 	size = ADMIN_CQ_SIZE(admin_queue->q_depth);  in ena_com_admin_destroy()
1719 	ena_dev->admin_queue.polling = polling;  in ena_com_set_admin_polling_mode()
1725 	ena_dev->admin_queue.auto_polling = polling;  in ena_com_set_admin_auto_polling_mode()
1788 	struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;  in ena_com_admin_init()  local
1805 	admin_queue->q_depth = ENA_ADMIN_QUEUE_DEPTH;  in ena_com_admin_init()
1807 	admin_queue->q_dmadev = ena_dev->dmadev;  in ena_com_admin_init()
1808 	admin_queue->polling = false;  in ena_com_admin_init()
1809 	admin_queue->curr_cmd_id = 0;  in ena_com_admin_init()
1811 	atomic_set(&admin_queue->outstanding_cmds, 0);  in ena_com_admin_init()
1813 	spin_lock_init(&admin_queue->q_lock);  in ena_com_admin_init()
1815 	ret = ena_com_init_comp_ctxt(admin_queue);  in ena_com_admin_init()
1819 	ret = ena_com_admin_init_sq(admin_queue);  in ena_com_admin_init()
1823 	ret = ena_com_admin_init_cq(admin_queue);  in ena_com_admin_init()
1827 	admin_queue->sq.db_addr = (u32 __iomem *)((uintptr_t)ena_dev->reg_bar +  in ena_com_admin_init()
1830 	addr_low = ENA_DMA_ADDR_TO_UINT32_LOW(admin_queue->sq.dma_addr);  in ena_com_admin_init()
1831 	addr_high = ENA_DMA_ADDR_TO_UINT32_HIGH(admin_queue->sq.dma_addr);  in ena_com_admin_init()
1836 	addr_low = ENA_DMA_ADDR_TO_UINT32_LOW(admin_queue->cq.dma_addr);  in ena_com_admin_init()
1837 	addr_high = ENA_DMA_ADDR_TO_UINT32_HIGH(admin_queue->cq.dma_addr);  in ena_com_admin_init()
1843 	aq_caps |= admin_queue->q_depth & ENA_REGS_AQ_CAPS_AQ_DEPTH_MASK;  in ena_com_admin_init()
1849 	acq_caps |= admin_queue->q_depth & ENA_REGS_ACQ_CAPS_ACQ_DEPTH_MASK;  in ena_com_admin_init()
1860 	admin_queue->ena_dev = ena_dev;  in ena_com_admin_init()
1861 	admin_queue->running_state = true;  in ena_com_admin_init()
2047 	ena_com_handle_admin_completion(&ena_dev->admin_queue);  in ena_com_admin_q_comp_intr_handler()
2186 		ena_dev->admin_queue.completion_timeout = timeout * 100000;  in ena_com_dev_reset()
2188 		ena_dev->admin_queue.completion_timeout = ADMIN_CMD_TIMEOUT_US;  in ena_com_dev_reset()
2199 	struct ena_com_admin_queue *admin_queue;  in ena_get_dev_stats()  local
2202 	admin_queue = &ena_dev->admin_queue;  in ena_get_dev_stats()
2208 	ret =  ena_com_execute_admin_command(admin_queue,  in ena_get_dev_stats()
2260 	struct ena_com_admin_queue *admin_queue;  in ena_com_set_dev_mtu()  local
2272 	admin_queue = &ena_dev->admin_queue;  in ena_com_set_dev_mtu()
2279 	ret = ena_com_execute_admin_command(admin_queue,  in ena_com_set_dev_mtu()
2313 	struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;  in ena_com_set_hash_function()  local
2359 	ret = ena_com_execute_admin_command(admin_queue,  in ena_com_set_hash_function()
2488 	struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;  in ena_com_set_hash_ctrl()  local
2521 	ret = ena_com_execute_admin_command(admin_queue,  in ena_com_set_hash_ctrl()
2652 	struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;  in ena_com_indirect_table_set()  local
2692 	ret = ena_com_execute_admin_command(admin_queue,  in ena_com_indirect_table_set()
2836 	struct ena_com_admin_queue *admin_queue;  in ena_com_set_host_attributes()  local
2847 	admin_queue = &ena_dev->admin_queue;  in ena_com_set_host_attributes()
2870 	ret = ena_com_execute_admin_command(admin_queue,  in ena_com_set_host_attributes()