706 	struct ath10k_ce_ring *dest_ring = pipe->dest_ring;  in __ath10k_ce_rx_num_free_bufs()  local
707 	unsigned int nentries_mask = dest_ring->nentries_mask;  in __ath10k_ce_rx_num_free_bufs()
708 	unsigned int write_index = dest_ring->write_index;  in __ath10k_ce_rx_num_free_bufs()
709 	unsigned int sw_index = dest_ring->sw_index;  in __ath10k_ce_rx_num_free_bufs()
722 	struct ath10k_ce_ring *dest_ring = pipe->dest_ring;  in __ath10k_ce_rx_post_buf()  local
723 	unsigned int nentries_mask = dest_ring->nentries_mask;  in __ath10k_ce_rx_post_buf()
724 	unsigned int write_index = dest_ring->write_index;  in __ath10k_ce_rx_post_buf()
725 	unsigned int sw_index = dest_ring->sw_index;  in __ath10k_ce_rx_post_buf()
726 	struct ce_desc *base = dest_ring->base_addr_owner_space;  in __ath10k_ce_rx_post_buf()
739 	dest_ring->per_transfer_context[write_index] = ctx;  in __ath10k_ce_rx_post_buf()
742 	dest_ring->write_index = write_index;  in __ath10k_ce_rx_post_buf()
753 	struct ath10k_ce_ring *dest_ring = pipe->dest_ring;  in __ath10k_ce_rx_post_buf_64()  local
754 	unsigned int nentries_mask = dest_ring->nentries_mask;  in __ath10k_ce_rx_post_buf_64()
755 	unsigned int write_index = dest_ring->write_index;  in __ath10k_ce_rx_post_buf_64()
756 	unsigned int sw_index = dest_ring->sw_index;  in __ath10k_ce_rx_post_buf_64()
757 	struct ce_desc_64 *base = dest_ring->base_addr_owner_space;  in __ath10k_ce_rx_post_buf_64()
772 	dest_ring->per_transfer_context[write_index] = ctx;  in __ath10k_ce_rx_post_buf_64()
775 	dest_ring->write_index = write_index;  in __ath10k_ce_rx_post_buf_64()
783 	struct ath10k_ce_ring *dest_ring = pipe->dest_ring;  in ath10k_ce_rx_update_write_idx()  local
784 	unsigned int nentries_mask = dest_ring->nentries_mask;  in ath10k_ce_rx_update_write_idx()
785 	unsigned int write_index = dest_ring->write_index;  in ath10k_ce_rx_update_write_idx()
792 	if (((cur_write_idx + nentries) & nentries_mask) == dest_ring->sw_index)  in ath10k_ce_rx_update_write_idx()
797 	dest_ring->write_index = write_index;  in ath10k_ce_rx_update_write_idx()
825 	struct ath10k_ce_ring *dest_ring = ce_state->dest_ring;  in _ath10k_ce_completed_recv_next_nolock()  local
826 	unsigned int nentries_mask = dest_ring->nentries_mask;  in _ath10k_ce_completed_recv_next_nolock()
827 	unsigned int sw_index = dest_ring->sw_index;  in _ath10k_ce_completed_recv_next_nolock()
829 	struct ce_desc *base = dest_ring->base_addr_owner_space;  in _ath10k_ce_completed_recv_next_nolock()
855 			dest_ring->per_transfer_context[sw_index];  in _ath10k_ce_completed_recv_next_nolock()
861 		dest_ring->per_transfer_context[sw_index] = NULL;  in _ath10k_ce_completed_recv_next_nolock()
865 	dest_ring->sw_index = sw_index;  in _ath10k_ce_completed_recv_next_nolock()
875 	struct ath10k_ce_ring *dest_ring = ce_state->dest_ring;  in _ath10k_ce_completed_recv_next_nolock_64()  local
876 	unsigned int nentries_mask = dest_ring->nentries_mask;  in _ath10k_ce_completed_recv_next_nolock_64()
877 	unsigned int sw_index = dest_ring->sw_index;  in _ath10k_ce_completed_recv_next_nolock_64()
878 	struct ce_desc_64 *base = dest_ring->base_addr_owner_space;  in _ath10k_ce_completed_recv_next_nolock_64()
904 			dest_ring->per_transfer_context[sw_index];  in _ath10k_ce_completed_recv_next_nolock_64()
910 		dest_ring->per_transfer_context[sw_index] = NULL;  in _ath10k_ce_completed_recv_next_nolock_64()
914 	dest_ring->sw_index = sw_index;  in _ath10k_ce_completed_recv_next_nolock_64()
952 	struct ath10k_ce_ring *dest_ring;  in _ath10k_ce_revoke_recv_next()  local
960 	dest_ring = ce_state->dest_ring;  in _ath10k_ce_revoke_recv_next()
962 	if (!dest_ring)  in _ath10k_ce_revoke_recv_next()
970 	nentries_mask = dest_ring->nentries_mask;  in _ath10k_ce_revoke_recv_next()
971 	sw_index = dest_ring->sw_index;  in _ath10k_ce_revoke_recv_next()
972 	write_index = dest_ring->write_index;  in _ath10k_ce_revoke_recv_next()
974 		struct ce_desc *base = dest_ring->base_addr_owner_space;  in _ath10k_ce_revoke_recv_next()
982 				dest_ring->per_transfer_context[sw_index];  in _ath10k_ce_revoke_recv_next()
985 		dest_ring->per_transfer_context[sw_index] = NULL;  in _ath10k_ce_revoke_recv_next()
990 		dest_ring->sw_index = sw_index;  in _ath10k_ce_revoke_recv_next()
1005 	struct ath10k_ce_ring *dest_ring;  in _ath10k_ce_revoke_recv_next_64()  local
1013 	dest_ring = ce_state->dest_ring;  in _ath10k_ce_revoke_recv_next_64()
1015 	if (!dest_ring)  in _ath10k_ce_revoke_recv_next_64()
1023 	nentries_mask = dest_ring->nentries_mask;  in _ath10k_ce_revoke_recv_next_64()
1024 	sw_index = dest_ring->sw_index;  in _ath10k_ce_revoke_recv_next_64()
1025 	write_index = dest_ring->write_index;  in _ath10k_ce_revoke_recv_next_64()
1027 		struct ce_desc_64 *base = dest_ring->base_addr_owner_space;  in _ath10k_ce_revoke_recv_next_64()
1036 				dest_ring->per_transfer_context[sw_index];  in _ath10k_ce_revoke_recv_next_64()
1039 		dest_ring->per_transfer_context[sw_index] = NULL;  in _ath10k_ce_revoke_recv_next_64()
1044 		dest_ring->sw_index = sw_index;  in _ath10k_ce_revoke_recv_next_64()
1469 	struct ath10k_ce_ring *dest_ring = ce_state->dest_ring;  in ath10k_ce_init_dest_ring()  local
1475 		memset(dest_ring->base_addr_owner_space, 0,  in ath10k_ce_init_dest_ring()
1478 		memset(dest_ring->base_addr_owner_space, 0,  in ath10k_ce_init_dest_ring()
1481 	dest_ring->sw_index = ath10k_ce_dest_ring_read_index_get(ar, ctrl_addr);  in ath10k_ce_init_dest_ring()
1482 	dest_ring->sw_index &= dest_ring->nentries_mask;  in ath10k_ce_init_dest_ring()
1483 	dest_ring->write_index =  in ath10k_ce_init_dest_ring()
1485 	dest_ring->write_index &= dest_ring->nentries_mask;  in ath10k_ce_init_dest_ring()
1488 					  dest_ring->base_addr_ce_space);  in ath10k_ce_init_dest_ring()
1496 		   ce_id, nentries, dest_ring->base_addr_owner_space);  in ath10k_ce_init_dest_ring()
1636 	struct ath10k_ce_ring *dest_ring;  in ath10k_ce_alloc_dest_ring()  local
1642 	dest_ring = kzalloc(struct_size(dest_ring, per_transfer_context,  in ath10k_ce_alloc_dest_ring()
1644 	if (dest_ring == NULL)  in ath10k_ce_alloc_dest_ring()
1647 	dest_ring->nentries = nentries;  in ath10k_ce_alloc_dest_ring()
1648 	dest_ring->nentries_mask = nentries - 1;  in ath10k_ce_alloc_dest_ring()
1654 	dest_ring->base_addr_owner_space_unaligned =  in ath10k_ce_alloc_dest_ring()
1659 	if (!dest_ring->base_addr_owner_space_unaligned) {  in ath10k_ce_alloc_dest_ring()
1660 		kfree(dest_ring);  in ath10k_ce_alloc_dest_ring()
1664 	dest_ring->base_addr_ce_space_unaligned = base_addr;  in ath10k_ce_alloc_dest_ring()
1666 	dest_ring->base_addr_owner_space =  in ath10k_ce_alloc_dest_ring()
1667 			PTR_ALIGN(dest_ring->base_addr_owner_space_unaligned,  in ath10k_ce_alloc_dest_ring()
1669 	dest_ring->base_addr_ce_space =  in ath10k_ce_alloc_dest_ring()
1670 				ALIGN(dest_ring->base_addr_ce_space_unaligned,  in ath10k_ce_alloc_dest_ring()
1673 	return dest_ring;  in ath10k_ce_alloc_dest_ring()
1680 	struct ath10k_ce_ring *dest_ring;  in ath10k_ce_alloc_dest_ring_64()  local
1686 	dest_ring = kzalloc(struct_size(dest_ring, per_transfer_context,  in ath10k_ce_alloc_dest_ring_64()
1688 	if (!dest_ring)  in ath10k_ce_alloc_dest_ring_64()
1691 	dest_ring->nentries = nentries;  in ath10k_ce_alloc_dest_ring_64()
1692 	dest_ring->nentries_mask = nentries - 1;  in ath10k_ce_alloc_dest_ring_64()
1697 	dest_ring->base_addr_owner_space_unaligned =  in ath10k_ce_alloc_dest_ring_64()
1702 	if (!dest_ring->base_addr_owner_space_unaligned) {  in ath10k_ce_alloc_dest_ring_64()
1703 		kfree(dest_ring);  in ath10k_ce_alloc_dest_ring_64()
1707 	dest_ring->base_addr_ce_space_unaligned = base_addr;  in ath10k_ce_alloc_dest_ring_64()
1712 	dest_ring->base_addr_owner_space =  in ath10k_ce_alloc_dest_ring_64()
1713 			PTR_ALIGN(dest_ring->base_addr_owner_space_unaligned,  in ath10k_ce_alloc_dest_ring_64()
1715 	dest_ring->base_addr_ce_space =  in ath10k_ce_alloc_dest_ring_64()
1716 			ALIGN(dest_ring->base_addr_ce_space_unaligned,  in ath10k_ce_alloc_dest_ring_64()
1719 	return dest_ring;  in ath10k_ce_alloc_dest_ring_64()
1799 	if (ce_state->dest_ring) {  in _ath10k_ce_free_pipe()
1801 				  (ce_state->dest_ring->nentries *  in _ath10k_ce_free_pipe()
1804 				  ce_state->dest_ring->base_addr_owner_space,  in _ath10k_ce_free_pipe()
1805 				  ce_state->dest_ring->base_addr_ce_space);  in _ath10k_ce_free_pipe()
1806 		kfree(ce_state->dest_ring);  in _ath10k_ce_free_pipe()
1810 	ce_state->dest_ring = NULL;  in _ath10k_ce_free_pipe()
1830 	if (ce_state->dest_ring) {  in _ath10k_ce_free_pipe_64()
1832 				  (ce_state->dest_ring->nentries *  in _ath10k_ce_free_pipe_64()
1835 				  ce_state->dest_ring->base_addr_owner_space,  in _ath10k_ce_free_pipe_64()
1836 				  ce_state->dest_ring->base_addr_ce_space);  in _ath10k_ce_free_pipe_64()
1837 		kfree(ce_state->dest_ring);  in _ath10k_ce_free_pipe_64()
1841 	ce_state->dest_ring = NULL;  in _ath10k_ce_free_pipe_64()
1980 		ce_state->dest_ring = ce_state->ops->ce_alloc_dst_ring(ar,  in ath10k_ce_alloc_pipe()
1983 		if (IS_ERR(ce_state->dest_ring)) {  in ath10k_ce_alloc_pipe()
1984 			ret = PTR_ERR(ce_state->dest_ring);  in ath10k_ce_alloc_pipe()
1987 			ce_state->dest_ring = NULL;  in ath10k_ce_alloc_pipe()