639 	struct ath10k_ce_ring *dest_ring = pipe->dest_ring;  in __ath10k_ce_rx_num_free_bufs()  local
640 	unsigned int nentries_mask = dest_ring->nentries_mask;  in __ath10k_ce_rx_num_free_bufs()
641 	unsigned int write_index = dest_ring->write_index;  in __ath10k_ce_rx_num_free_bufs()
642 	unsigned int sw_index = dest_ring->sw_index;  in __ath10k_ce_rx_num_free_bufs()
655 	struct ath10k_ce_ring *dest_ring = pipe->dest_ring;  in __ath10k_ce_rx_post_buf()  local
656 	unsigned int nentries_mask = dest_ring->nentries_mask;  in __ath10k_ce_rx_post_buf()
657 	unsigned int write_index = dest_ring->write_index;  in __ath10k_ce_rx_post_buf()
658 	unsigned int sw_index = dest_ring->sw_index;  in __ath10k_ce_rx_post_buf()
659 	struct ce_desc *base = dest_ring->base_addr_owner_space;  in __ath10k_ce_rx_post_buf()
672 	dest_ring->per_transfer_context[write_index] = ctx;  in __ath10k_ce_rx_post_buf()
675 	dest_ring->write_index = write_index;  in __ath10k_ce_rx_post_buf()
686 	struct ath10k_ce_ring *dest_ring = pipe->dest_ring;  in __ath10k_ce_rx_post_buf_64()  local
687 	unsigned int nentries_mask = dest_ring->nentries_mask;  in __ath10k_ce_rx_post_buf_64()
688 	unsigned int write_index = dest_ring->write_index;  in __ath10k_ce_rx_post_buf_64()
689 	unsigned int sw_index = dest_ring->sw_index;  in __ath10k_ce_rx_post_buf_64()
690 	struct ce_desc_64 *base = dest_ring->base_addr_owner_space;  in __ath10k_ce_rx_post_buf_64()
705 	dest_ring->per_transfer_context[write_index] = ctx;  in __ath10k_ce_rx_post_buf_64()
708 	dest_ring->write_index = write_index;  in __ath10k_ce_rx_post_buf_64()
716 	struct ath10k_ce_ring *dest_ring = pipe->dest_ring;  in ath10k_ce_rx_update_write_idx()  local
717 	unsigned int nentries_mask = dest_ring->nentries_mask;  in ath10k_ce_rx_update_write_idx()
718 	unsigned int write_index = dest_ring->write_index;  in ath10k_ce_rx_update_write_idx()
725 	if (((cur_write_idx + nentries) & nentries_mask) == dest_ring->sw_index)  in ath10k_ce_rx_update_write_idx()
730 	dest_ring->write_index = write_index;  in ath10k_ce_rx_update_write_idx()
758 	struct ath10k_ce_ring *dest_ring = ce_state->dest_ring;  in _ath10k_ce_completed_recv_next_nolock()  local
759 	unsigned int nentries_mask = dest_ring->nentries_mask;  in _ath10k_ce_completed_recv_next_nolock()
760 	unsigned int sw_index = dest_ring->sw_index;  in _ath10k_ce_completed_recv_next_nolock()
762 	struct ce_desc *base = dest_ring->base_addr_owner_space;  in _ath10k_ce_completed_recv_next_nolock()
788 			dest_ring->per_transfer_context[sw_index];  in _ath10k_ce_completed_recv_next_nolock()
794 		dest_ring->per_transfer_context[sw_index] = NULL;  in _ath10k_ce_completed_recv_next_nolock()
798 	dest_ring->sw_index = sw_index;  in _ath10k_ce_completed_recv_next_nolock()
808 	struct ath10k_ce_ring *dest_ring = ce_state->dest_ring;  in _ath10k_ce_completed_recv_next_nolock_64()  local
809 	unsigned int nentries_mask = dest_ring->nentries_mask;  in _ath10k_ce_completed_recv_next_nolock_64()
810 	unsigned int sw_index = dest_ring->sw_index;  in _ath10k_ce_completed_recv_next_nolock_64()
811 	struct ce_desc_64 *base = dest_ring->base_addr_owner_space;  in _ath10k_ce_completed_recv_next_nolock_64()
837 			dest_ring->per_transfer_context[sw_index];  in _ath10k_ce_completed_recv_next_nolock_64()
843 		dest_ring->per_transfer_context[sw_index] = NULL;  in _ath10k_ce_completed_recv_next_nolock_64()
847 	dest_ring->sw_index = sw_index;  in _ath10k_ce_completed_recv_next_nolock_64()
885 	struct ath10k_ce_ring *dest_ring;  in _ath10k_ce_revoke_recv_next()  local
893 	dest_ring = ce_state->dest_ring;  in _ath10k_ce_revoke_recv_next()
895 	if (!dest_ring)  in _ath10k_ce_revoke_recv_next()
903 	nentries_mask = dest_ring->nentries_mask;  in _ath10k_ce_revoke_recv_next()
904 	sw_index = dest_ring->sw_index;  in _ath10k_ce_revoke_recv_next()
905 	write_index = dest_ring->write_index;  in _ath10k_ce_revoke_recv_next()
907 		struct ce_desc *base = dest_ring->base_addr_owner_space;  in _ath10k_ce_revoke_recv_next()
915 				dest_ring->per_transfer_context[sw_index];  in _ath10k_ce_revoke_recv_next()
918 		dest_ring->per_transfer_context[sw_index] = NULL;  in _ath10k_ce_revoke_recv_next()
923 		dest_ring->sw_index = sw_index;  in _ath10k_ce_revoke_recv_next()
938 	struct ath10k_ce_ring *dest_ring;  in _ath10k_ce_revoke_recv_next_64()  local
946 	dest_ring = ce_state->dest_ring;  in _ath10k_ce_revoke_recv_next_64()
948 	if (!dest_ring)  in _ath10k_ce_revoke_recv_next_64()
956 	nentries_mask = dest_ring->nentries_mask;  in _ath10k_ce_revoke_recv_next_64()
957 	sw_index = dest_ring->sw_index;  in _ath10k_ce_revoke_recv_next_64()
958 	write_index = dest_ring->write_index;  in _ath10k_ce_revoke_recv_next_64()
960 		struct ce_desc_64 *base = dest_ring->base_addr_owner_space;  in _ath10k_ce_revoke_recv_next_64()
969 				dest_ring->per_transfer_context[sw_index];  in _ath10k_ce_revoke_recv_next_64()
972 		dest_ring->per_transfer_context[sw_index] = NULL;  in _ath10k_ce_revoke_recv_next_64()
977 		dest_ring->sw_index = sw_index;  in _ath10k_ce_revoke_recv_next_64()
1402 	struct ath10k_ce_ring *dest_ring = ce_state->dest_ring;  in ath10k_ce_init_dest_ring()  local
1408 		memset(dest_ring->base_addr_owner_space, 0,  in ath10k_ce_init_dest_ring()
1411 		memset(dest_ring->base_addr_owner_space, 0,  in ath10k_ce_init_dest_ring()
1414 	dest_ring->sw_index = ath10k_ce_dest_ring_read_index_get(ar, ctrl_addr);  in ath10k_ce_init_dest_ring()
1415 	dest_ring->sw_index &= dest_ring->nentries_mask;  in ath10k_ce_init_dest_ring()
1416 	dest_ring->write_index =  in ath10k_ce_init_dest_ring()
1418 	dest_ring->write_index &= dest_ring->nentries_mask;  in ath10k_ce_init_dest_ring()
1421 					  dest_ring->base_addr_ce_space);  in ath10k_ce_init_dest_ring()
1429 		   ce_id, nentries, dest_ring->base_addr_owner_space);  in ath10k_ce_init_dest_ring()
1569 	struct ath10k_ce_ring *dest_ring;  in ath10k_ce_alloc_dest_ring()  local
1575 	dest_ring = kzalloc(struct_size(dest_ring, per_transfer_context,  in ath10k_ce_alloc_dest_ring()
1577 	if (dest_ring == NULL)  in ath10k_ce_alloc_dest_ring()
1580 	dest_ring->nentries = nentries;  in ath10k_ce_alloc_dest_ring()
1581 	dest_ring->nentries_mask = nentries - 1;  in ath10k_ce_alloc_dest_ring()
1587 	dest_ring->base_addr_owner_space_unaligned =  in ath10k_ce_alloc_dest_ring()
1592 	if (!dest_ring->base_addr_owner_space_unaligned) {  in ath10k_ce_alloc_dest_ring()
1593 		kfree(dest_ring);  in ath10k_ce_alloc_dest_ring()
1597 	dest_ring->base_addr_ce_space_unaligned = base_addr;  in ath10k_ce_alloc_dest_ring()
1599 	dest_ring->base_addr_owner_space =  in ath10k_ce_alloc_dest_ring()
1600 			PTR_ALIGN(dest_ring->base_addr_owner_space_unaligned,  in ath10k_ce_alloc_dest_ring()
1602 	dest_ring->base_addr_ce_space =  in ath10k_ce_alloc_dest_ring()
1603 				ALIGN(dest_ring->base_addr_ce_space_unaligned,  in ath10k_ce_alloc_dest_ring()
1606 	return dest_ring;  in ath10k_ce_alloc_dest_ring()
1613 	struct ath10k_ce_ring *dest_ring;  in ath10k_ce_alloc_dest_ring_64()  local
1619 	dest_ring = kzalloc(struct_size(dest_ring, per_transfer_context,  in ath10k_ce_alloc_dest_ring_64()
1621 	if (!dest_ring)  in ath10k_ce_alloc_dest_ring_64()
1624 	dest_ring->nentries = nentries;  in ath10k_ce_alloc_dest_ring_64()
1625 	dest_ring->nentries_mask = nentries - 1;  in ath10k_ce_alloc_dest_ring_64()
1630 	dest_ring->base_addr_owner_space_unaligned =  in ath10k_ce_alloc_dest_ring_64()
1635 	if (!dest_ring->base_addr_owner_space_unaligned) {  in ath10k_ce_alloc_dest_ring_64()
1636 		kfree(dest_ring);  in ath10k_ce_alloc_dest_ring_64()
1640 	dest_ring->base_addr_ce_space_unaligned = base_addr;  in ath10k_ce_alloc_dest_ring_64()
1645 	dest_ring->base_addr_owner_space =  in ath10k_ce_alloc_dest_ring_64()
1646 			PTR_ALIGN(dest_ring->base_addr_owner_space_unaligned,  in ath10k_ce_alloc_dest_ring_64()
1648 	dest_ring->base_addr_ce_space =  in ath10k_ce_alloc_dest_ring_64()
1649 			ALIGN(dest_ring->base_addr_ce_space_unaligned,  in ath10k_ce_alloc_dest_ring_64()
1652 	return dest_ring;  in ath10k_ce_alloc_dest_ring_64()
1732 	if (ce_state->dest_ring) {  in _ath10k_ce_free_pipe()
1734 				  (ce_state->dest_ring->nentries *  in _ath10k_ce_free_pipe()
1737 				  ce_state->dest_ring->base_addr_owner_space,  in _ath10k_ce_free_pipe()
1738 				  ce_state->dest_ring->base_addr_ce_space);  in _ath10k_ce_free_pipe()
1739 		kfree(ce_state->dest_ring);  in _ath10k_ce_free_pipe()
1743 	ce_state->dest_ring = NULL;  in _ath10k_ce_free_pipe()
1763 	if (ce_state->dest_ring) {  in _ath10k_ce_free_pipe_64()
1765 				  (ce_state->dest_ring->nentries *  in _ath10k_ce_free_pipe_64()
1768 				  ce_state->dest_ring->base_addr_owner_space,  in _ath10k_ce_free_pipe_64()
1769 				  ce_state->dest_ring->base_addr_ce_space);  in _ath10k_ce_free_pipe_64()
1770 		kfree(ce_state->dest_ring);  in _ath10k_ce_free_pipe_64()
1774 	ce_state->dest_ring = NULL;  in _ath10k_ce_free_pipe_64()
1913 		ce_state->dest_ring = ce_state->ops->ce_alloc_dst_ring(ar,  in ath10k_ce_alloc_pipe()
1916 		if (IS_ERR(ce_state->dest_ring)) {  in ath10k_ce_alloc_pipe()
1917 			ret = PTR_ERR(ce_state->dest_ring);  in ath10k_ce_alloc_pipe()
1920 			ce_state->dest_ring = NULL;  in ath10k_ce_alloc_pipe()