39 static void  __check_db_timeout(struct octeon_device *oct, u64 iq_no);
43 static inline int IQ_INSTR_MODE_64B(struct octeon_device *oct, int iq_no)  in IQ_INSTR_MODE_64B()  argument
46 	    (struct octeon_instr_queue *)oct->instr_queue[iq_no];  in IQ_INSTR_MODE_64B()
50 #define IQ_INSTR_MODE_32B(oct, iq_no)  (!IQ_INSTR_MODE_64B(oct, iq_no))  argument
56 int octeon_init_instr_queue(struct octeon_device *oct,  in octeon_init_instr_queue()  argument
65 	int numa_node = dev_to_node(&oct->pci_dev->dev);  in octeon_init_instr_queue()
67 	if (OCTEON_CN6XXX(oct))  in octeon_init_instr_queue()
68 		conf = &(CFG_GET_IQ_CFG(CHIP_CONF(oct, cn6xxx)));  in octeon_init_instr_queue()
69 	else if (OCTEON_CN23XX_PF(oct))  in octeon_init_instr_queue()
70 		conf = &(CFG_GET_IQ_CFG(CHIP_CONF(oct, cn23xx_pf)));  in octeon_init_instr_queue()
71 	else if (OCTEON_CN23XX_VF(oct))  in octeon_init_instr_queue()
72 		conf = &(CFG_GET_IQ_CFG(CHIP_CONF(oct, cn23xx_vf)));  in octeon_init_instr_queue()
75 		dev_err(&oct->pci_dev->dev, "Unsupported Chip %x\n",  in octeon_init_instr_queue()
76 			oct->chip_id);  in octeon_init_instr_queue()
82 	iq = oct->instr_queue[iq_no];  in octeon_init_instr_queue()
84 	iq->oct_dev = oct;  in octeon_init_instr_queue()
86 	iq->base_addr = lio_dma_alloc(oct, q_size, &iq->base_addr_dma);  in octeon_init_instr_queue()
88 		dev_err(&oct->pci_dev->dev, "Cannot allocate memory for instr queue %d\n",  in octeon_init_instr_queue()
103 		lio_dma_free(oct, q_size, iq->base_addr, iq->base_addr_dma);  in octeon_init_instr_queue()
104 		dev_err(&oct->pci_dev->dev, "Alloc failed for IQ[%d] nr free list\n",  in octeon_init_instr_queue()
109 	dev_dbg(&oct->pci_dev->dev, "IQ[%d]: base: %p basedma: %pad count: %d\n",  in octeon_init_instr_queue()
135 	oct->io_qmask.iq |= BIT_ULL(iq_no);  in octeon_init_instr_queue()
138 	oct->io_qmask.iq64B |= ((conf->instr_type == 64) << iq_no);  in octeon_init_instr_queue()
141 	oct->fn_list.setup_iq_regs(oct, iq_no);  in octeon_init_instr_queue()
143 	oct->check_db_wq[iq_no].wq = alloc_workqueue("check_iq_db",  in octeon_init_instr_queue()
146 	if (!oct->check_db_wq[iq_no].wq) {  in octeon_init_instr_queue()
149 		lio_dma_free(oct, q_size, iq->base_addr, iq->base_addr_dma);  in octeon_init_instr_queue()
150 		dev_err(&oct->pci_dev->dev, "check db wq create failed for iq %d\n",  in octeon_init_instr_queue()
155 	db_wq = &oct->check_db_wq[iq_no];  in octeon_init_instr_queue()
158 	db_wq->wk.ctxptr = oct;  in octeon_init_instr_queue()
165 int octeon_delete_instr_queue(struct octeon_device *oct, u32 iq_no)  in octeon_delete_instr_queue()  argument
168 	struct octeon_instr_queue *iq = oct->instr_queue[iq_no];  in octeon_delete_instr_queue()
170 	cancel_delayed_work_sync(&oct->check_db_wq[iq_no].wk.work);  in octeon_delete_instr_queue()
171 	destroy_workqueue(oct->check_db_wq[iq_no].wq);  in octeon_delete_instr_queue()
173 	if (OCTEON_CN6XXX(oct))  in octeon_delete_instr_queue()
175 		    CFG_GET_IQ_INSTR_TYPE(CHIP_CONF(oct, cn6xxx));  in octeon_delete_instr_queue()
176 	else if (OCTEON_CN23XX_PF(oct))  in octeon_delete_instr_queue()
178 		    CFG_GET_IQ_INSTR_TYPE(CHIP_CONF(oct, cn23xx_pf));  in octeon_delete_instr_queue()
179 	else if (OCTEON_CN23XX_VF(oct))  in octeon_delete_instr_queue()
181 		    CFG_GET_IQ_INSTR_TYPE(CHIP_CONF(oct, cn23xx_vf));  in octeon_delete_instr_queue()
187 		lio_dma_free(oct, (u32)q_size, iq->base_addr,  in octeon_delete_instr_queue()
189 		oct->io_qmask.iq &= ~(1ULL << iq_no);  in octeon_delete_instr_queue()
190 		vfree(oct->instr_queue[iq_no]);  in octeon_delete_instr_queue()
191 		oct->instr_queue[iq_no] = NULL;  in octeon_delete_instr_queue()
192 		oct->num_iqs--;  in octeon_delete_instr_queue()
199 int octeon_setup_iq(struct octeon_device *oct,  in octeon_setup_iq()  argument
207 	int numa_node = dev_to_node(&oct->pci_dev->dev);  in octeon_setup_iq()
209 	if (oct->instr_queue[iq_no]) {  in octeon_setup_iq()
210 		dev_dbg(&oct->pci_dev->dev, "IQ is in use. Cannot create the IQ: %d again\n",  in octeon_setup_iq()
212 		oct->instr_queue[iq_no]->txpciq.u64 = txpciq.u64;  in octeon_setup_iq()
213 		oct->instr_queue[iq_no]->app_ctx = app_ctx;  in octeon_setup_iq()
216 	oct->instr_queue[iq_no] =  in octeon_setup_iq()
218 	if (!oct->instr_queue[iq_no])  in octeon_setup_iq()
219 		oct->instr_queue[iq_no] =  in octeon_setup_iq()
221 	if (!oct->instr_queue[iq_no])  in octeon_setup_iq()
225 	oct->instr_queue[iq_no]->q_index = q_index;  in octeon_setup_iq()
226 	oct->instr_queue[iq_no]->app_ctx = app_ctx;  in octeon_setup_iq()
227 	oct->instr_queue[iq_no]->ifidx = ifidx;  in octeon_setup_iq()
229 	if (octeon_init_instr_queue(oct, txpciq, num_descs)) {  in octeon_setup_iq()
230 		vfree(oct->instr_queue[iq_no]);  in octeon_setup_iq()
231 		oct->instr_queue[iq_no] = NULL;  in octeon_setup_iq()
235 	oct->num_iqs++;  in octeon_setup_iq()
236 	if (oct->fn_list.enable_io_queues(oct)) {  in octeon_setup_iq()
237 		octeon_delete_instr_queue(oct, iq_no);  in octeon_setup_iq()
244 int lio_wait_for_instr_fetch(struct octeon_device *oct)  in lio_wait_for_instr_fetch()  argument
251 		for (i = 0; i < MAX_OCTEON_INSTR_QUEUES(oct); i++) {  in lio_wait_for_instr_fetch()
252 			if (!(oct->io_qmask.iq & BIT_ULL(i)))  in lio_wait_for_instr_fetch()
255 			    atomic_read(&oct->instr_queue[i]->instr_pending);  in lio_wait_for_instr_fetch()
257 				__check_db_timeout(oct, i);  in lio_wait_for_instr_fetch()
272 ring_doorbell(struct octeon_device *oct, struct octeon_instr_queue *iq)  in ring_doorbell()  argument
274 	if (atomic_read(&oct->status) == OCT_DEV_RUNNING) {  in ring_doorbell()
284 octeon_ring_doorbell_locked(struct octeon_device *oct, u32 iq_no)  in octeon_ring_doorbell_locked()  argument
288 	iq = oct->instr_queue[iq_no];  in octeon_ring_doorbell_locked()
291 		ring_doorbell(oct, iq);  in octeon_ring_doorbell_locked()
344 octeon_register_reqtype_free_fn(struct octeon_device *oct, int reqtype,  in octeon_register_reqtype_free_fn()  argument
348 		dev_err(&oct->pci_dev->dev, "%s: Invalid reqtype: %d\n",  in octeon_register_reqtype_free_fn()
353 	reqtype_free_fn[oct->octeon_id][reqtype] = fn;  in octeon_register_reqtype_free_fn()
368 lio_process_iq_request_list(struct octeon_device *oct,  in lio_process_iq_request_list()  argument
371 	struct cavium_wq *cwq = &oct->dma_comp_wq;  in lio_process_iq_request_list()
394 			reqtype_free_fn[oct->octeon_id][reqtype](buf);  in lio_process_iq_request_list()
405 			spin_lock_irqsave(&oct->response_list  in lio_process_iq_request_list()
407 			atomic_inc(&oct->response_list  in lio_process_iq_request_list()
409 			list_add_tail(&sc->node, &oct->response_list  in lio_process_iq_request_list()
411 			spin_unlock_irqrestore(&oct->response_list  in lio_process_iq_request_list()
416 			dev_err(&oct->pci_dev->dev,  in lio_process_iq_request_list()
436 	if (atomic_read(&oct->response_list  in lio_process_iq_request_list()
445 octeon_flush_iq(struct octeon_device *oct, struct octeon_instr_queue *iq,  in octeon_flush_iq()  argument
457 	iq->octeon_read_index = oct->fn_list.update_iq_read_idx(iq);  in octeon_flush_iq()
466 				lio_process_iq_request_list(oct, iq,  in octeon_flush_iq()
471 				lio_process_iq_request_list(oct, iq, 0);  in octeon_flush_iq()
497 static void __check_db_timeout(struct octeon_device *oct, u64 iq_no)  in __check_db_timeout()  argument
502 	if (!oct)  in __check_db_timeout()
505 	iq = oct->instr_queue[iq_no];  in __check_db_timeout()
519 	octeon_flush_iq(oct, iq, 0);  in __check_db_timeout()
530 	struct octeon_device *oct = (struct octeon_device *)wk->ctxptr;  in check_db_timeout()  local
532 	struct cavium_wq *db_wq = &oct->check_db_wq[iq_no];  in check_db_timeout()
535 	__check_db_timeout(oct, iq_no);  in check_db_timeout()
540 octeon_send_command(struct octeon_device *oct, u32 iq_no,  in octeon_send_command()  argument
546 	struct octeon_instr_queue *iq = oct->instr_queue[iq_no];  in octeon_send_command()
559 		INCR_INSTRQUEUE_PKT_COUNT(oct, iq_no, bytes_sent, datasize);  in octeon_send_command()
560 		INCR_INSTRQUEUE_PKT_COUNT(oct, iq_no, instr_posted, 1);  in octeon_send_command()
564 			ring_doorbell(oct, iq);  in octeon_send_command()
566 		INCR_INSTRQUEUE_PKT_COUNT(oct, iq_no, instr_dropped, 1);  in octeon_send_command()
580 octeon_prepare_soft_command(struct octeon_device *oct,  in octeon_prepare_soft_command()  argument
598 	oct_cfg = octeon_get_conf(oct);  in octeon_prepare_soft_command()
600 	if (OCTEON_CN23XX_PF(oct) || OCTEON_CN23XX_VF(oct)) {  in octeon_prepare_soft_command()
603 		ih3->pkind = oct->instr_queue[sc->iq_no]->txpciq.s.pkind;  in octeon_prepare_soft_command()
611 			oct->instr_queue[sc->iq_no]->txpciq.s.use_qpg;  in octeon_prepare_soft_command()
616 			oct->instr_queue[sc->iq_no]->txpciq.s.ctrl_qpg;  in octeon_prepare_soft_command()
635 			rdp->pcie_port = oct->pcie_port;  in octeon_prepare_soft_command()
672 			rdp->pcie_port = oct->pcie_port;  in octeon_prepare_soft_command()
686 int octeon_send_soft_command(struct octeon_device *oct,  in octeon_send_soft_command()  argument
695 	iq = oct->instr_queue[sc->iq_no];  in octeon_send_soft_command()
697 		dev_err(&oct->pci_dev->dev, "Soft commands are not allowed on Queue %d\n",  in octeon_send_soft_command()
699 		INCR_INSTRQUEUE_PKT_COUNT(oct, sc->iq_no, instr_dropped, 1);  in octeon_send_soft_command()
703 	if (OCTEON_CN23XX_PF(oct) || OCTEON_CN23XX_VF(oct)) {  in octeon_send_soft_command()
735 	return (octeon_send_command(oct, sc->iq_no, 1, &sc->cmd, sc,  in octeon_send_soft_command()
739 int octeon_setup_sc_buffer_pool(struct octeon_device *oct)  in octeon_setup_sc_buffer_pool()  argument
745 	INIT_LIST_HEAD(&oct->sc_buf_pool.head);  in octeon_setup_sc_buffer_pool()
746 	spin_lock_init(&oct->sc_buf_pool.lock);  in octeon_setup_sc_buffer_pool()
747 	atomic_set(&oct->sc_buf_pool.alloc_buf_count, 0);  in octeon_setup_sc_buffer_pool()
751 			lio_dma_alloc(oct,  in octeon_setup_sc_buffer_pool()
755 			octeon_free_sc_buffer_pool(oct);  in octeon_setup_sc_buffer_pool()
762 		list_add_tail(&sc->node, &oct->sc_buf_pool.head);  in octeon_setup_sc_buffer_pool()
768 int octeon_free_sc_done_list(struct octeon_device *oct)  in octeon_free_sc_done_list()  argument
775 	done_sc_list = &oct->response_list[OCTEON_DONE_SC_LIST];  in octeon_free_sc_done_list()
776 	zombie_sc_list = &oct->response_list[OCTEON_ZOMBIE_SC_LIST];  in octeon_free_sc_done_list()
781 	sc_lists_lock = &oct->response_list[OCTEON_ORDERED_SC_LIST].lock;  in octeon_free_sc_done_list()
797 				octeon_free_soft_command(oct, sc);  in octeon_free_sc_done_list()
807 int octeon_free_sc_zombie_list(struct octeon_device *oct)  in octeon_free_sc_zombie_list()  argument
814 	zombie_sc_list = &oct->response_list[OCTEON_ZOMBIE_SC_LIST];  in octeon_free_sc_zombie_list()
815 	sc_lists_lock = &oct->response_list[OCTEON_ORDERED_SC_LIST].lock;  in octeon_free_sc_zombie_list()
823 		octeon_free_soft_command(oct, sc);  in octeon_free_sc_zombie_list()
831 int octeon_free_sc_buffer_pool(struct octeon_device *oct)  in octeon_free_sc_buffer_pool()  argument
836 	octeon_free_sc_zombie_list(oct);  in octeon_free_sc_buffer_pool()
838 	spin_lock_bh(&oct->sc_buf_pool.lock);  in octeon_free_sc_buffer_pool()
840 	list_for_each_safe(tmp, tmp2, &oct->sc_buf_pool.head) {  in octeon_free_sc_buffer_pool()
845 		lio_dma_free(oct, sc->size, sc, sc->dma_addr);  in octeon_free_sc_buffer_pool()
848 	INIT_LIST_HEAD(&oct->sc_buf_pool.head);  in octeon_free_sc_buffer_pool()
850 	spin_unlock_bh(&oct->sc_buf_pool.lock);  in octeon_free_sc_buffer_pool()
855 struct octeon_soft_command *octeon_alloc_soft_command(struct octeon_device *oct,  in octeon_alloc_soft_command()  argument
872 	spin_lock_bh(&oct->sc_buf_pool.lock);  in octeon_alloc_soft_command()
874 	if (list_empty(&oct->sc_buf_pool.head)) {  in octeon_alloc_soft_command()
875 		spin_unlock_bh(&oct->sc_buf_pool.lock);  in octeon_alloc_soft_command()
879 	list_for_each(tmp, &oct->sc_buf_pool.head)  in octeon_alloc_soft_command()
884 	atomic_inc(&oct->sc_buf_pool.alloc_buf_count);  in octeon_alloc_soft_command()
886 	spin_unlock_bh(&oct->sc_buf_pool.lock);  in octeon_alloc_soft_command()
926 void octeon_free_soft_command(struct octeon_device *oct,  in octeon_free_soft_command()  argument
929 	spin_lock_bh(&oct->sc_buf_pool.lock);  in octeon_free_soft_command()
931 	list_add_tail(&sc->node, &oct->sc_buf_pool.head);  in octeon_free_soft_command()
933 	atomic_dec(&oct->sc_buf_pool.alloc_buf_count);  in octeon_free_soft_command()
935 	spin_unlock_bh(&oct->sc_buf_pool.lock);  in octeon_free_soft_command()