52 					struct ath10k_ce_pipe *ce_state)  in shadow_sr_wr_ind_addr()  argument
54 	u32 ce_id = ce_state->id;  in shadow_sr_wr_ind_addr()
81 					 struct ath10k_ce_pipe *ce_state)  in shadow_dst_wr_ind_addr()  argument
83 	u32 ce_id = ce_state->id;  in shadow_dst_wr_ind_addr()
190 	struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];  in ath10k_ce_src_ring_read_index_get()  local
194 	    (ce_state->attr_flags & CE_ATTR_DIS_INTR))  in ath10k_ce_src_ring_read_index_get()
205 					  struct ath10k_ce_pipe *ce_state,  in ath10k_ce_shadow_src_ring_write_index_set()  argument
208 	ath10k_ce_write32(ar, shadow_sr_wr_ind_addr(ar, ce_state), value);  in ath10k_ce_shadow_src_ring_write_index_set()
213 					   struct ath10k_ce_pipe *ce_state,  in ath10k_ce_shadow_dest_ring_write_index_set()  argument
216 	ath10k_ce_write32(ar, shadow_dst_wr_ind_addr(ar, ce_state), value);  in ath10k_ce_shadow_dest_ring_write_index_set()
224 	struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];  in ath10k_ce_src_ring_base_addr_set()  local
231 	if (ce_state->ops->ce_set_src_ring_base_addr_hi) {  in ath10k_ce_src_ring_base_addr_set()
232 		ce_state->ops->ce_set_src_ring_base_addr_hi(ar, ce_ctrl_addr,  in ath10k_ce_src_ring_base_addr_set()
311 	struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];  in ath10k_ce_dest_ring_read_index_get()  local
315 	    (ce_state->attr_flags & CE_ATTR_DIS_INTR))  in ath10k_ce_dest_ring_read_index_get()
329 	struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];  in ath10k_ce_dest_ring_base_addr_set()  local
336 	if (ce_state->ops->ce_set_dest_ring_base_addr_hi) {  in ath10k_ce_dest_ring_base_addr_set()
337 		ce_state->ops->ce_set_dest_ring_base_addr_hi(ar, ce_ctrl_addr,  in ath10k_ce_dest_ring_base_addr_set()
488 static int _ath10k_ce_send_nolock(struct ath10k_ce_pipe *ce_state,  in _ath10k_ce_send_nolock()  argument
495 	struct ath10k *ar = ce_state->ar;  in _ath10k_ce_send_nolock()
496 	struct ath10k_ce_ring *src_ring = ce_state->src_ring;  in _ath10k_ce_send_nolock()
501 	u32 ctrl_addr = ce_state->ctrl_addr;  in _ath10k_ce_send_nolock()
505 	if (nbytes > ce_state->src_sz_max)  in _ath10k_ce_send_nolock()
507 			    __func__, nbytes, ce_state->src_sz_max);  in _ath10k_ce_send_nolock()
545 static int _ath10k_ce_send_nolock_64(struct ath10k_ce_pipe *ce_state,  in _ath10k_ce_send_nolock_64()  argument
552 	struct ath10k *ar = ce_state->ar;  in _ath10k_ce_send_nolock_64()
553 	struct ath10k_ce_ring *src_ring = ce_state->src_ring;  in _ath10k_ce_send_nolock_64()
558 	u32 ctrl_addr = ce_state->ctrl_addr;  in _ath10k_ce_send_nolock_64()
566 	if (nbytes > ce_state->src_sz_max)  in _ath10k_ce_send_nolock_64()
568 			    __func__, nbytes, ce_state->src_sz_max);  in _ath10k_ce_send_nolock_64()
571 		sw_index = ath10k_ce_src_ring_read_index_from_ddr(ar, ce_state->id);  in _ath10k_ce_send_nolock_64()
614 			ath10k_ce_shadow_src_ring_write_index_set(ar, ce_state,  in _ath10k_ce_send_nolock_64()
626 int ath10k_ce_send_nolock(struct ath10k_ce_pipe *ce_state,  in ath10k_ce_send_nolock()  argument
633 	return ce_state->ops->ce_send_nolock(ce_state, per_transfer_context,  in ath10k_ce_send_nolock()
666 int ath10k_ce_send(struct ath10k_ce_pipe *ce_state,  in ath10k_ce_send()  argument
673 	struct ath10k *ar = ce_state->ar;  in ath10k_ce_send()
678 	ret = ath10k_ce_send_nolock(ce_state, per_transfer_context,  in ath10k_ce_send()
821 	 _ath10k_ce_completed_recv_next_nolock(struct ath10k_ce_pipe *ce_state,  in _ath10k_ce_completed_recv_next_nolock()  argument
825 	struct ath10k_ce_ring *dest_ring = ce_state->dest_ring;  in _ath10k_ce_completed_recv_next_nolock()
860 	if (ce_state->id != 5)  in _ath10k_ce_completed_recv_next_nolock()
871 _ath10k_ce_completed_recv_next_nolock_64(struct ath10k_ce_pipe *ce_state,  in _ath10k_ce_completed_recv_next_nolock_64()  argument
875 	struct ath10k_ce_ring *dest_ring = ce_state->dest_ring;  in _ath10k_ce_completed_recv_next_nolock_64()
909 	if (ce_state->id != 5)  in _ath10k_ce_completed_recv_next_nolock_64()
919 int ath10k_ce_completed_recv_next_nolock(struct ath10k_ce_pipe *ce_state,  in ath10k_ce_completed_recv_next_nolock()  argument
923 	return ce_state->ops->ce_completed_recv_next_nolock(ce_state,  in ath10k_ce_completed_recv_next_nolock()
929 int ath10k_ce_completed_recv_next(struct ath10k_ce_pipe *ce_state,  in ath10k_ce_completed_recv_next()  argument
933 	struct ath10k *ar = ce_state->ar;  in ath10k_ce_completed_recv_next()
938 	ret = ce_state->ops->ce_completed_recv_next_nolock(ce_state,  in ath10k_ce_completed_recv_next()
948 static int _ath10k_ce_revoke_recv_next(struct ath10k_ce_pipe *ce_state,  in _ath10k_ce_revoke_recv_next()  argument
960 	dest_ring = ce_state->dest_ring;  in _ath10k_ce_revoke_recv_next()
965 	ar = ce_state->ar;  in _ath10k_ce_revoke_recv_next()
1001 static int _ath10k_ce_revoke_recv_next_64(struct ath10k_ce_pipe *ce_state,  in _ath10k_ce_revoke_recv_next_64()  argument
1013 	dest_ring = ce_state->dest_ring;  in _ath10k_ce_revoke_recv_next_64()
1018 	ar = ce_state->ar;  in _ath10k_ce_revoke_recv_next_64()
1055 int ath10k_ce_revoke_recv_next(struct ath10k_ce_pipe *ce_state,  in ath10k_ce_revoke_recv_next()  argument
1059 	return ce_state->ops->ce_revoke_recv_next(ce_state,  in ath10k_ce_revoke_recv_next()
1069 static int _ath10k_ce_completed_send_next_nolock(struct ath10k_ce_pipe *ce_state,  in _ath10k_ce_completed_send_next_nolock()  argument
1072 	struct ath10k_ce_ring *src_ring = ce_state->src_ring;  in _ath10k_ce_completed_send_next_nolock()
1073 	u32 ctrl_addr = ce_state->ctrl_addr;  in _ath10k_ce_completed_send_next_nolock()
1074 	struct ath10k *ar = ce_state->ar;  in _ath10k_ce_completed_send_next_nolock()
1122 static int _ath10k_ce_completed_send_next_nolock_64(struct ath10k_ce_pipe *ce_state,  in _ath10k_ce_completed_send_next_nolock_64()  argument
1125 	struct ath10k_ce_ring *src_ring = ce_state->src_ring;  in _ath10k_ce_completed_send_next_nolock_64()
1126 	u32 ctrl_addr = ce_state->ctrl_addr;  in _ath10k_ce_completed_send_next_nolock_64()
1127 	struct ath10k *ar = ce_state->ar;  in _ath10k_ce_completed_send_next_nolock_64()
1175 int ath10k_ce_completed_send_next_nolock(struct ath10k_ce_pipe *ce_state,  in ath10k_ce_completed_send_next_nolock()  argument
1178 	return ce_state->ops->ce_completed_send_next_nolock(ce_state,  in ath10k_ce_completed_send_next_nolock()
1219 int ath10k_ce_cancel_send_next(struct ath10k_ce_pipe *ce_state,  in ath10k_ce_cancel_send_next()  argument
1233 	src_ring = ce_state->src_ring;  in ath10k_ce_cancel_send_next()
1238 	ar = ce_state->ar;  in ath10k_ce_cancel_send_next()
1248 		ce_state->ops->ce_extract_desc_data(ar, src_ring, sw_index,  in ath10k_ce_cancel_send_next()
1273 int ath10k_ce_completed_send_next(struct ath10k_ce_pipe *ce_state,  in ath10k_ce_completed_send_next()  argument
1276 	struct ath10k *ar = ce_state->ar;  in ath10k_ce_completed_send_next()
1281 	ret = ath10k_ce_completed_send_next_nolock(ce_state,  in ath10k_ce_completed_send_next()
1298 	struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];  in ath10k_ce_per_engine_service()  local
1300 	u32 ctrl_addr = ce_state->ctrl_addr;  in ath10k_ce_per_engine_service()
1315 	if (ce_state->recv_cb)  in ath10k_ce_per_engine_service()
1316 		ce_state->recv_cb(ce_state);  in ath10k_ce_per_engine_service()
1318 	if (ce_state->send_cb)  in ath10k_ce_per_engine_service()
1319 		ce_state->send_cb(ce_state);  in ath10k_ce_per_engine_service()
1355 static void ath10k_ce_per_engine_handler_adjust(struct ath10k_ce_pipe *ce_state)  in ath10k_ce_per_engine_handler_adjust()  argument
1357 	u32 ctrl_addr = ce_state->ctrl_addr;  in ath10k_ce_per_engine_handler_adjust()
1358 	struct ath10k *ar = ce_state->ar;  in ath10k_ce_per_engine_handler_adjust()
1359 	bool disable_copy_compl_intr = ce_state->attr_flags & CE_ATTR_DIS_INTR;  in ath10k_ce_per_engine_handler_adjust()
1362 	    (ce_state->send_cb || ce_state->recv_cb))  in ath10k_ce_per_engine_handler_adjust()
1373 	struct ath10k_ce_pipe *ce_state;  in ath10k_ce_disable_interrupt()  local
1376 	ce_state  = &ce->ce_states[ce_id];  in ath10k_ce_disable_interrupt()
1377 	if (ce_state->attr_flags & CE_ATTR_POLL)  in ath10k_ce_disable_interrupt()
1400 	struct ath10k_ce_pipe *ce_state;  in ath10k_ce_enable_interrupt()  local
1402 	ce_state  = &ce->ce_states[ce_id];  in ath10k_ce_enable_interrupt()
1403 	if (ce_state->attr_flags & CE_ATTR_POLL)  in ath10k_ce_enable_interrupt()
1406 	ath10k_ce_per_engine_handler_adjust(ce_state);  in ath10k_ce_enable_interrupt()
1427 	struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];  in ath10k_ce_init_src_ring()  local
1428 	struct ath10k_ce_ring *src_ring = ce_state->src_ring;  in ath10k_ce_init_src_ring()
1468 	struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];  in ath10k_ce_init_dest_ring()  local
1469 	struct ath10k_ce_ring *dest_ring = ce_state->dest_ring;  in ath10k_ce_init_dest_ring()
1785 	struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];  in _ath10k_ce_free_pipe()  local
1787 	if (ce_state->src_ring) {  in _ath10k_ce_free_pipe()
1789 			kfree(ce_state->src_ring->shadow_base_unaligned);  in _ath10k_ce_free_pipe()
1791 				  (ce_state->src_ring->nentries *  in _ath10k_ce_free_pipe()
1794 				  ce_state->src_ring->base_addr_owner_space,  in _ath10k_ce_free_pipe()
1795 				  ce_state->src_ring->base_addr_ce_space);  in _ath10k_ce_free_pipe()
1796 		kfree(ce_state->src_ring);  in _ath10k_ce_free_pipe()
1799 	if (ce_state->dest_ring) {  in _ath10k_ce_free_pipe()
1801 				  (ce_state->dest_ring->nentries *  in _ath10k_ce_free_pipe()
1804 				  ce_state->dest_ring->base_addr_owner_space,  in _ath10k_ce_free_pipe()
1805 				  ce_state->dest_ring->base_addr_ce_space);  in _ath10k_ce_free_pipe()
1806 		kfree(ce_state->dest_ring);  in _ath10k_ce_free_pipe()
1809 	ce_state->src_ring = NULL;  in _ath10k_ce_free_pipe()
1810 	ce_state->dest_ring = NULL;  in _ath10k_ce_free_pipe()
1816 	struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];  in _ath10k_ce_free_pipe_64()  local
1818 	if (ce_state->src_ring) {  in _ath10k_ce_free_pipe_64()
1820 			kfree(ce_state->src_ring->shadow_base_unaligned);  in _ath10k_ce_free_pipe_64()
1822 				  (ce_state->src_ring->nentries *  in _ath10k_ce_free_pipe_64()
1825 				  ce_state->src_ring->base_addr_owner_space,  in _ath10k_ce_free_pipe_64()
1826 				  ce_state->src_ring->base_addr_ce_space);  in _ath10k_ce_free_pipe_64()
1827 		kfree(ce_state->src_ring);  in _ath10k_ce_free_pipe_64()
1830 	if (ce_state->dest_ring) {  in _ath10k_ce_free_pipe_64()
1832 				  (ce_state->dest_ring->nentries *  in _ath10k_ce_free_pipe_64()
1835 				  ce_state->dest_ring->base_addr_owner_space,  in _ath10k_ce_free_pipe_64()
1836 				  ce_state->dest_ring->base_addr_ce_space);  in _ath10k_ce_free_pipe_64()
1837 		kfree(ce_state->dest_ring);  in _ath10k_ce_free_pipe_64()
1840 	ce_state->src_ring = NULL;  in _ath10k_ce_free_pipe_64()
1841 	ce_state->dest_ring = NULL;  in _ath10k_ce_free_pipe_64()
1847 	struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];  in ath10k_ce_free_pipe()  local
1849 	ce_state->ops->ce_free_pipe(ar, ce_id);  in ath10k_ce_free_pipe()
1923 			      struct ath10k_ce_pipe *ce_state)  in ath10k_ce_set_ops()  argument
1927 		ce_state->ops = &ce_64_ops;  in ath10k_ce_set_ops()
1930 		ce_state->ops = &ce_ops;  in ath10k_ce_set_ops()
1939 	struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];  in ath10k_ce_alloc_pipe()  local
1942 	ath10k_ce_set_ops(ar, ce_state);  in ath10k_ce_alloc_pipe()
1955 	ce_state->ar = ar;  in ath10k_ce_alloc_pipe()
1956 	ce_state->id = ce_id;  in ath10k_ce_alloc_pipe()
1957 	ce_state->ctrl_addr = ath10k_ce_base_address(ar, ce_id);  in ath10k_ce_alloc_pipe()
1958 	ce_state->attr_flags = attr->flags;  in ath10k_ce_alloc_pipe()
1959 	ce_state->src_sz_max = attr->src_sz_max;  in ath10k_ce_alloc_pipe()
1962 		ce_state->send_cb = attr->send_cb;  in ath10k_ce_alloc_pipe()
1965 		ce_state->recv_cb = attr->recv_cb;  in ath10k_ce_alloc_pipe()
1968 		ce_state->src_ring =  in ath10k_ce_alloc_pipe()
1969 			ce_state->ops->ce_alloc_src_ring(ar, ce_id, attr);  in ath10k_ce_alloc_pipe()
1970 		if (IS_ERR(ce_state->src_ring)) {  in ath10k_ce_alloc_pipe()
1971 			ret = PTR_ERR(ce_state->src_ring);  in ath10k_ce_alloc_pipe()
1974 			ce_state->src_ring = NULL;  in ath10k_ce_alloc_pipe()
1980 		ce_state->dest_ring = ce_state->ops->ce_alloc_dst_ring(ar,  in ath10k_ce_alloc_pipe()
1983 		if (IS_ERR(ce_state->dest_ring)) {  in ath10k_ce_alloc_pipe()
1984 			ret = PTR_ERR(ce_state->dest_ring);  in ath10k_ce_alloc_pipe()
1987 			ce_state->dest_ring = NULL;  in ath10k_ce_alloc_pipe()