63 					struct ath10k_ce_pipe *ce_state)  in shadow_sr_wr_ind_addr()  argument
65 	u32 ce_id = ce_state->id;  in shadow_sr_wr_ind_addr()
92 					 struct ath10k_ce_pipe *ce_state)  in shadow_dst_wr_ind_addr()  argument
94 	u32 ce_id = ce_state->id;  in shadow_dst_wr_ind_addr()
201 	struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];  in ath10k_ce_src_ring_read_index_get()  local
205 	    (ce_state->attr_flags & CE_ATTR_DIS_INTR))  in ath10k_ce_src_ring_read_index_get()
216 					  struct ath10k_ce_pipe *ce_state,  in ath10k_ce_shadow_src_ring_write_index_set()  argument
219 	ath10k_ce_write32(ar, shadow_sr_wr_ind_addr(ar, ce_state), value);  in ath10k_ce_shadow_src_ring_write_index_set()
224 					   struct ath10k_ce_pipe *ce_state,  in ath10k_ce_shadow_dest_ring_write_index_set()  argument
227 	ath10k_ce_write32(ar, shadow_dst_wr_ind_addr(ar, ce_state), value);  in ath10k_ce_shadow_dest_ring_write_index_set()
302 	struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];  in ath10k_ce_dest_ring_read_index_get()  local
306 	    (ce_state->attr_flags & CE_ATTR_DIS_INTR))  in ath10k_ce_dest_ring_read_index_get()
454 static int _ath10k_ce_send_nolock(struct ath10k_ce_pipe *ce_state,  in _ath10k_ce_send_nolock()  argument
461 	struct ath10k *ar = ce_state->ar;  in _ath10k_ce_send_nolock()
462 	struct ath10k_ce_ring *src_ring = ce_state->src_ring;  in _ath10k_ce_send_nolock()
467 	u32 ctrl_addr = ce_state->ctrl_addr;  in _ath10k_ce_send_nolock()
471 	if (nbytes > ce_state->src_sz_max)  in _ath10k_ce_send_nolock()
473 			    __func__, nbytes, ce_state->src_sz_max);  in _ath10k_ce_send_nolock()
505 			ath10k_ce_shadow_src_ring_write_index_set(ar, ce_state,  in _ath10k_ce_send_nolock()
517 static int _ath10k_ce_send_nolock_64(struct ath10k_ce_pipe *ce_state,  in _ath10k_ce_send_nolock_64()  argument
524 	struct ath10k *ar = ce_state->ar;  in _ath10k_ce_send_nolock_64()
525 	struct ath10k_ce_ring *src_ring = ce_state->src_ring;  in _ath10k_ce_send_nolock_64()
530 	u32 ctrl_addr = ce_state->ctrl_addr;  in _ath10k_ce_send_nolock_64()
538 	if (nbytes > ce_state->src_sz_max)  in _ath10k_ce_send_nolock_64()
540 			    __func__, nbytes, ce_state->src_sz_max);  in _ath10k_ce_send_nolock_64()
543 		sw_index = ath10k_ce_src_ring_read_index_from_ddr(ar, ce_state->id);  in _ath10k_ce_send_nolock_64()
592 int ath10k_ce_send_nolock(struct ath10k_ce_pipe *ce_state,  in ath10k_ce_send_nolock()  argument
599 	return ce_state->ops->ce_send_nolock(ce_state, per_transfer_context,  in ath10k_ce_send_nolock()
632 int ath10k_ce_send(struct ath10k_ce_pipe *ce_state,  in ath10k_ce_send()  argument
639 	struct ath10k *ar = ce_state->ar;  in ath10k_ce_send()
644 	ret = ath10k_ce_send_nolock(ce_state, per_transfer_context,  in ath10k_ce_send()
787 	 _ath10k_ce_completed_recv_next_nolock(struct ath10k_ce_pipe *ce_state,  in _ath10k_ce_completed_recv_next_nolock()  argument
791 	struct ath10k_ce_ring *dest_ring = ce_state->dest_ring;  in _ath10k_ce_completed_recv_next_nolock()
826 	if (ce_state->id != 5)  in _ath10k_ce_completed_recv_next_nolock()
837 _ath10k_ce_completed_recv_next_nolock_64(struct ath10k_ce_pipe *ce_state,  in _ath10k_ce_completed_recv_next_nolock_64()  argument
841 	struct ath10k_ce_ring *dest_ring = ce_state->dest_ring;  in _ath10k_ce_completed_recv_next_nolock_64()
875 	if (ce_state->id != 5)  in _ath10k_ce_completed_recv_next_nolock_64()
885 int ath10k_ce_completed_recv_next_nolock(struct ath10k_ce_pipe *ce_state,  in ath10k_ce_completed_recv_next_nolock()  argument
889 	return ce_state->ops->ce_completed_recv_next_nolock(ce_state,  in ath10k_ce_completed_recv_next_nolock()
895 int ath10k_ce_completed_recv_next(struct ath10k_ce_pipe *ce_state,  in ath10k_ce_completed_recv_next()  argument
899 	struct ath10k *ar = ce_state->ar;  in ath10k_ce_completed_recv_next()
904 	ret = ce_state->ops->ce_completed_recv_next_nolock(ce_state,  in ath10k_ce_completed_recv_next()
914 static int _ath10k_ce_revoke_recv_next(struct ath10k_ce_pipe *ce_state,  in _ath10k_ce_revoke_recv_next()  argument
926 	dest_ring = ce_state->dest_ring;  in _ath10k_ce_revoke_recv_next()
931 	ar = ce_state->ar;  in _ath10k_ce_revoke_recv_next()
967 static int _ath10k_ce_revoke_recv_next_64(struct ath10k_ce_pipe *ce_state,  in _ath10k_ce_revoke_recv_next_64()  argument
979 	dest_ring = ce_state->dest_ring;  in _ath10k_ce_revoke_recv_next_64()
984 	ar = ce_state->ar;  in _ath10k_ce_revoke_recv_next_64()
1021 int ath10k_ce_revoke_recv_next(struct ath10k_ce_pipe *ce_state,  in ath10k_ce_revoke_recv_next()  argument
1025 	return ce_state->ops->ce_revoke_recv_next(ce_state,  in ath10k_ce_revoke_recv_next()
1035 int ath10k_ce_completed_send_next_nolock(struct ath10k_ce_pipe *ce_state,  in ath10k_ce_completed_send_next_nolock()  argument
1038 	struct ath10k_ce_ring *src_ring = ce_state->src_ring;  in ath10k_ce_completed_send_next_nolock()
1039 	u32 ctrl_addr = ce_state->ctrl_addr;  in ath10k_ce_completed_send_next_nolock()
1040 	struct ath10k *ar = ce_state->ar;  in ath10k_ce_completed_send_next_nolock()
1125 int ath10k_ce_cancel_send_next(struct ath10k_ce_pipe *ce_state,  in ath10k_ce_cancel_send_next()  argument
1139 	src_ring = ce_state->src_ring;  in ath10k_ce_cancel_send_next()
1144 	ar = ce_state->ar;  in ath10k_ce_cancel_send_next()
1154 		ce_state->ops->ce_extract_desc_data(ar, src_ring, sw_index,  in ath10k_ce_cancel_send_next()
1179 int ath10k_ce_completed_send_next(struct ath10k_ce_pipe *ce_state,  in ath10k_ce_completed_send_next()  argument
1182 	struct ath10k *ar = ce_state->ar;  in ath10k_ce_completed_send_next()
1187 	ret = ath10k_ce_completed_send_next_nolock(ce_state,  in ath10k_ce_completed_send_next()
1204 	struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];  in ath10k_ce_per_engine_service()  local
1206 	u32 ctrl_addr = ce_state->ctrl_addr;  in ath10k_ce_per_engine_service()
1216 	if (ce_state->recv_cb)  in ath10k_ce_per_engine_service()
1217 		ce_state->recv_cb(ce_state);  in ath10k_ce_per_engine_service()
1219 	if (ce_state->send_cb)  in ath10k_ce_per_engine_service()
1220 		ce_state->send_cb(ce_state);  in ath10k_ce_per_engine_service()
1266 static void ath10k_ce_per_engine_handler_adjust(struct ath10k_ce_pipe *ce_state)  in ath10k_ce_per_engine_handler_adjust()  argument
1268 	u32 ctrl_addr = ce_state->ctrl_addr;  in ath10k_ce_per_engine_handler_adjust()
1269 	struct ath10k *ar = ce_state->ar;  in ath10k_ce_per_engine_handler_adjust()
1270 	bool disable_copy_compl_intr = ce_state->attr_flags & CE_ATTR_DIS_INTR;  in ath10k_ce_per_engine_handler_adjust()
1273 	    (ce_state->send_cb || ce_state->recv_cb))  in ath10k_ce_per_engine_handler_adjust()
1301 	struct ath10k_ce_pipe *ce_state;  in ath10k_ce_enable_interrupts()  local
1307 		ce_state  = &ce->ce_states[ce_id];  in ath10k_ce_enable_interrupts()
1308 		ath10k_ce_per_engine_handler_adjust(ce_state);  in ath10k_ce_enable_interrupts()
1318 	struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];  in ath10k_ce_init_src_ring()  local
1319 	struct ath10k_ce_ring *src_ring = ce_state->src_ring;  in ath10k_ce_init_src_ring()
1359 	struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];  in ath10k_ce_init_dest_ring()  local
1360 	struct ath10k_ce_ring *dest_ring = ce_state->dest_ring;  in ath10k_ce_init_dest_ring()
1687 	struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];  in _ath10k_ce_free_pipe()  local
1689 	if (ce_state->src_ring) {  in _ath10k_ce_free_pipe()
1691 			kfree(ce_state->src_ring->shadow_base_unaligned);  in _ath10k_ce_free_pipe()
1693 				  (ce_state->src_ring->nentries *  in _ath10k_ce_free_pipe()
1696 				  ce_state->src_ring->base_addr_owner_space,  in _ath10k_ce_free_pipe()
1697 				  ce_state->src_ring->base_addr_ce_space);  in _ath10k_ce_free_pipe()
1698 		kfree(ce_state->src_ring);  in _ath10k_ce_free_pipe()
1701 	if (ce_state->dest_ring) {  in _ath10k_ce_free_pipe()
1703 				  (ce_state->dest_ring->nentries *  in _ath10k_ce_free_pipe()
1706 				  ce_state->dest_ring->base_addr_owner_space,  in _ath10k_ce_free_pipe()
1707 				  ce_state->dest_ring->base_addr_ce_space);  in _ath10k_ce_free_pipe()
1708 		kfree(ce_state->dest_ring);  in _ath10k_ce_free_pipe()
1711 	ce_state->src_ring = NULL;  in _ath10k_ce_free_pipe()
1712 	ce_state->dest_ring = NULL;  in _ath10k_ce_free_pipe()
1718 	struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];  in _ath10k_ce_free_pipe_64()  local
1720 	if (ce_state->src_ring) {  in _ath10k_ce_free_pipe_64()
1722 			kfree(ce_state->src_ring->shadow_base_unaligned);  in _ath10k_ce_free_pipe_64()
1724 				  (ce_state->src_ring->nentries *  in _ath10k_ce_free_pipe_64()
1727 				  ce_state->src_ring->base_addr_owner_space,  in _ath10k_ce_free_pipe_64()
1728 				  ce_state->src_ring->base_addr_ce_space);  in _ath10k_ce_free_pipe_64()
1729 		kfree(ce_state->src_ring);  in _ath10k_ce_free_pipe_64()
1732 	if (ce_state->dest_ring) {  in _ath10k_ce_free_pipe_64()
1734 				  (ce_state->dest_ring->nentries *  in _ath10k_ce_free_pipe_64()
1737 				  ce_state->dest_ring->base_addr_owner_space,  in _ath10k_ce_free_pipe_64()
1738 				  ce_state->dest_ring->base_addr_ce_space);  in _ath10k_ce_free_pipe_64()
1739 		kfree(ce_state->dest_ring);  in _ath10k_ce_free_pipe_64()
1742 	ce_state->src_ring = NULL;  in _ath10k_ce_free_pipe_64()
1743 	ce_state->dest_ring = NULL;  in _ath10k_ce_free_pipe_64()
1749 	struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];  in ath10k_ce_free_pipe()  local
1751 	ce_state->ops->ce_free_pipe(ar, ce_id);  in ath10k_ce_free_pipe()
1819 			      struct ath10k_ce_pipe *ce_state)  in ath10k_ce_set_ops()  argument
1823 		ce_state->ops = &ce_64_ops;  in ath10k_ce_set_ops()
1826 		ce_state->ops = &ce_ops;  in ath10k_ce_set_ops()
1835 	struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];  in ath10k_ce_alloc_pipe()  local
1838 	ath10k_ce_set_ops(ar, ce_state);  in ath10k_ce_alloc_pipe()
1851 	ce_state->ar = ar;  in ath10k_ce_alloc_pipe()
1852 	ce_state->id = ce_id;  in ath10k_ce_alloc_pipe()
1853 	ce_state->ctrl_addr = ath10k_ce_base_address(ar, ce_id);  in ath10k_ce_alloc_pipe()
1854 	ce_state->attr_flags = attr->flags;  in ath10k_ce_alloc_pipe()
1855 	ce_state->src_sz_max = attr->src_sz_max;  in ath10k_ce_alloc_pipe()
1858 		ce_state->send_cb = attr->send_cb;  in ath10k_ce_alloc_pipe()
1861 		ce_state->recv_cb = attr->recv_cb;  in ath10k_ce_alloc_pipe()
1864 		ce_state->src_ring =  in ath10k_ce_alloc_pipe()
1865 			ce_state->ops->ce_alloc_src_ring(ar, ce_id, attr);  in ath10k_ce_alloc_pipe()
1866 		if (IS_ERR(ce_state->src_ring)) {  in ath10k_ce_alloc_pipe()
1867 			ret = PTR_ERR(ce_state->src_ring);  in ath10k_ce_alloc_pipe()
1870 			ce_state->src_ring = NULL;  in ath10k_ce_alloc_pipe()
1876 		ce_state->dest_ring = ce_state->ops->ce_alloc_dst_ring(ar,  in ath10k_ce_alloc_pipe()
1879 		if (IS_ERR(ce_state->dest_ring)) {  in ath10k_ce_alloc_pipe()
1880 			ret = PTR_ERR(ce_state->dest_ring);  in ath10k_ce_alloc_pipe()
1883 			ce_state->dest_ring = NULL;  in ath10k_ce_alloc_pipe()