251 void ath11k_ce_stop_shadow_timers(struct ath11k_base *ab)  in ath11k_ce_stop_shadow_timers()  argument
255 	if (!ab->hw_params.supports_shadow_regs)  in ath11k_ce_stop_shadow_timers()
258 	for (i = 0; i < ab->hw_params.ce_count; i++)  in ath11k_ce_stop_shadow_timers()
260 			ath11k_dp_shadow_stop_timer(ab, &ab->ce.hp_timer[i]);  in ath11k_ce_stop_shadow_timers()
266 	struct ath11k_base *ab = pipe->ab;  in ath11k_ce_rx_buf_enqueue_pipe()  local
274 	lockdep_assert_held(&ab->ce.ce_lock);  in ath11k_ce_rx_buf_enqueue_pipe()
278 	srng = &ab->hal.srng_list[ring->hal_ring_id];  in ath11k_ce_rx_buf_enqueue_pipe()
282 	ath11k_hal_srng_access_begin(ab, srng);  in ath11k_ce_rx_buf_enqueue_pipe()
284 	if (unlikely(ath11k_hal_srng_src_num_free(ab, srng, false) < 1)) {  in ath11k_ce_rx_buf_enqueue_pipe()
289 	desc = ath11k_hal_srng_src_get_next_entry(ab, srng);  in ath11k_ce_rx_buf_enqueue_pipe()
305 	ath11k_hal_srng_access_end(ab, srng);  in ath11k_ce_rx_buf_enqueue_pipe()
314 	struct ath11k_base *ab = pipe->ab;  in ath11k_ce_rx_post_pipe()  local
322 	spin_lock_bh(&ab->ce.ce_lock);  in ath11k_ce_rx_post_pipe()
332 		paddr = dma_map_single(ab->dev, skb->data,  in ath11k_ce_rx_post_pipe()
335 		if (unlikely(dma_mapping_error(ab->dev, paddr))) {  in ath11k_ce_rx_post_pipe()
336 			ath11k_warn(ab, "failed to dma map ce rx buf\n");  in ath11k_ce_rx_post_pipe()
347 			ath11k_warn(ab, "failed to enqueue rx buf: %d\n", ret);  in ath11k_ce_rx_post_pipe()
348 			dma_unmap_single(ab->dev, paddr,  in ath11k_ce_rx_post_pipe()
357 	spin_unlock_bh(&ab->ce.ce_lock);  in ath11k_ce_rx_post_pipe()
364 	struct ath11k_base *ab = pipe->ab;  in ath11k_ce_completed_recv_next()  local
371 	spin_lock_bh(&ab->ce.ce_lock);  in ath11k_ce_completed_recv_next()
376 	srng = &ab->hal.srng_list[pipe->status_ring->hal_ring_id];  in ath11k_ce_completed_recv_next()
380 	ath11k_hal_srng_access_begin(ab, srng);  in ath11k_ce_completed_recv_next()
382 	desc = ath11k_hal_srng_dst_get_next_entry(ab, srng);  in ath11k_ce_completed_recv_next()
402 	ath11k_hal_srng_access_end(ab, srng);  in ath11k_ce_completed_recv_next()
406 	spin_unlock_bh(&ab->ce.ce_lock);  in ath11k_ce_completed_recv_next()
413 	struct ath11k_base *ab = pipe->ab;  in ath11k_ce_recv_process_cb()  local
422 		dma_unmap_single(ab->dev, ATH11K_SKB_RXCB(skb)->paddr,  in ath11k_ce_recv_process_cb()
426 			ath11k_warn(ab, "rxed more than expected (nbytes %d, max %d)",  in ath11k_ce_recv_process_cb()
437 		ath11k_dbg(ab, ATH11K_DBG_AHB, "rx ce pipe %d len %d\n",  in ath11k_ce_recv_process_cb()
439 		pipe->recv_cb(ab, skb);  in ath11k_ce_recv_process_cb()
444 		ath11k_warn(ab, "failed to post rx buf to pipe: %d err: %d\n",  in ath11k_ce_recv_process_cb()
446 		mod_timer(&ab->rx_replenish_retry,  in ath11k_ce_recv_process_cb()
453 	struct ath11k_base *ab = pipe->ab;  in ath11k_ce_completed_send_next()  local
460 	spin_lock_bh(&ab->ce.ce_lock);  in ath11k_ce_completed_send_next()
465 	srng = &ab->hal.srng_list[pipe->src_ring->hal_ring_id];  in ath11k_ce_completed_send_next()
469 	ath11k_hal_srng_access_begin(ab, srng);  in ath11k_ce_completed_send_next()
471 	desc = ath11k_hal_srng_src_reap_next(ab, srng);  in ath11k_ce_completed_send_next()
487 	spin_unlock_bh(&ab->ce.ce_lock);  in ath11k_ce_completed_send_next()
494 	struct ath11k_base *ab = pipe->ab;  in ath11k_ce_send_done_cb()  local
501 		dma_unmap_single(ab->dev, ATH11K_SKB_CB(skb)->paddr, skb->len,  in ath11k_ce_send_done_cb()
507 static void ath11k_ce_srng_msi_ring_params_setup(struct ath11k_base *ab, u32 ce_id,  in ath11k_ce_srng_msi_ring_params_setup()  argument
517 	ret = ath11k_get_user_msi_vector(ab, "CE",  in ath11k_ce_srng_msi_ring_params_setup()
524 	ath11k_get_msi_address(ab, &addr_lo, &addr_hi);  in ath11k_ce_srng_msi_ring_params_setup()
525 	ath11k_get_ce_msi_idx(ab, ce_id, &msi_data_idx);  in ath11k_ce_srng_msi_ring_params_setup()
533 static int ath11k_ce_init_ring(struct ath11k_base *ab,  in ath11k_ce_init_ring()  argument
544 	if (!(CE_ATTR_DIS_INTR & ab->hw_params.host_ce_config[ce_id].flags))  in ath11k_ce_init_ring()
545 		ath11k_ce_srng_msi_ring_params_setup(ab, ce_id, &params);  in ath11k_ce_init_ring()
549 		if (!(CE_ATTR_DIS_INTR & ab->hw_params.host_ce_config[ce_id].flags))  in ath11k_ce_init_ring()
553 		params.max_buffer_len = ab->hw_params.host_ce_config[ce_id].src_sz_max;  in ath11k_ce_init_ring()
554 		if (!(ab->hw_params.host_ce_config[ce_id].flags & CE_ATTR_DIS_INTR)) {  in ath11k_ce_init_ring()
561 		if (!(ab->hw_params.host_ce_config[ce_id].flags & CE_ATTR_DIS_INTR)) {  in ath11k_ce_init_ring()
567 		ath11k_warn(ab, "Invalid CE ring type %d\n", type);  in ath11k_ce_init_ring()
573 	ret = ath11k_hal_srng_setup(ab, type, ce_id, 0, &params);  in ath11k_ce_init_ring()
575 		ath11k_warn(ab, "failed to setup srng: %d ring_id %d\n",  in ath11k_ce_init_ring()
582 	if (ab->hw_params.supports_shadow_regs &&  in ath11k_ce_init_ring()
584 		ath11k_dp_shadow_init_timer(ab, &ab->ce.hp_timer[ce_id],  in ath11k_ce_init_ring()
592 ath11k_ce_alloc_ring(struct ath11k_base *ab, int nentries, int desc_sz)  in ath11k_ce_alloc_ring()  argument
608 		dma_alloc_coherent(ab->dev,  in ath11k_ce_alloc_ring()
628 static int ath11k_ce_alloc_pipe(struct ath11k_base *ab, int ce_id)  in ath11k_ce_alloc_pipe()  argument
630 	struct ath11k_ce_pipe *pipe = &ab->ce.ce_pipe[ce_id];  in ath11k_ce_alloc_pipe()
631 	const struct ce_attr *attr = &ab->hw_params.host_ce_config[ce_id];  in ath11k_ce_alloc_pipe()
642 		ring = ath11k_ce_alloc_ring(ab, nentries, desc_sz);  in ath11k_ce_alloc_pipe()
652 		ring = ath11k_ce_alloc_ring(ab, nentries, desc_sz);  in ath11k_ce_alloc_pipe()
658 		ring = ath11k_ce_alloc_ring(ab, nentries, desc_sz);  in ath11k_ce_alloc_pipe()
667 void ath11k_ce_per_engine_service(struct ath11k_base *ab, u16 ce_id)  in ath11k_ce_per_engine_service()  argument
669 	struct ath11k_ce_pipe *pipe = &ab->ce.ce_pipe[ce_id];  in ath11k_ce_per_engine_service()
678 void ath11k_ce_poll_send_completed(struct ath11k_base *ab, u8 pipe_id)  in ath11k_ce_poll_send_completed()  argument
680 	struct ath11k_ce_pipe *pipe = &ab->ce.ce_pipe[pipe_id];  in ath11k_ce_poll_send_completed()
687 int ath11k_ce_send(struct ath11k_base *ab, struct sk_buff *skb, u8 pipe_id,  in ath11k_ce_send()  argument
690 	struct ath11k_ce_pipe *pipe = &ab->ce.ce_pipe[pipe_id];  in ath11k_ce_send()
704 		spin_lock_bh(&ab->ce.ce_lock);  in ath11k_ce_send()
715 		spin_unlock_bh(&ab->ce.ce_lock);  in ath11k_ce_send()
718 			ath11k_ce_poll_send_completed(ab, pipe->pipe_num);  in ath11k_ce_send()
721 	if (test_bit(ATH11K_FLAG_CRASH_FLUSH, &ab->dev_flags))  in ath11k_ce_send()
724 	spin_lock_bh(&ab->ce.ce_lock);  in ath11k_ce_send()
729 	srng = &ab->hal.srng_list[pipe->src_ring->hal_ring_id];  in ath11k_ce_send()
733 	ath11k_hal_srng_access_begin(ab, srng);  in ath11k_ce_send()
735 	if (unlikely(ath11k_hal_srng_src_num_free(ab, srng, false) < 1)) {  in ath11k_ce_send()
736 		ath11k_hal_srng_access_end(ab, srng);  in ath11k_ce_send()
741 	desc = ath11k_hal_srng_src_get_next_reaped(ab, srng);  in ath11k_ce_send()
743 		ath11k_hal_srng_access_end(ab, srng);  in ath11k_ce_send()
758 	ath11k_hal_srng_access_end(ab, srng);  in ath11k_ce_send()
761 		ath11k_dp_shadow_start_timer(ab, srng, &ab->ce.hp_timer[pipe_id]);  in ath11k_ce_send()
765 	spin_unlock_bh(&ab->ce.ce_lock);  in ath11k_ce_send()
772 	spin_unlock_bh(&ab->ce.ce_lock);  in ath11k_ce_send()
779 	struct ath11k_base *ab = pipe->ab;  in ath11k_ce_rx_pipe_cleanup()  local
793 		dma_unmap_single(ab->dev, ATH11K_SKB_RXCB(skb)->paddr,  in ath11k_ce_rx_pipe_cleanup()
799 static void ath11k_ce_shadow_config(struct ath11k_base *ab)  in ath11k_ce_shadow_config()  argument
803 	for (i = 0; i < ab->hw_params.ce_count; i++) {  in ath11k_ce_shadow_config()
804 		if (ab->hw_params.host_ce_config[i].src_nentries)  in ath11k_ce_shadow_config()
805 			ath11k_hal_srng_update_shadow_config(ab,  in ath11k_ce_shadow_config()
808 		if (ab->hw_params.host_ce_config[i].dest_nentries) {  in ath11k_ce_shadow_config()
809 			ath11k_hal_srng_update_shadow_config(ab,  in ath11k_ce_shadow_config()
812 			ath11k_hal_srng_update_shadow_config(ab,  in ath11k_ce_shadow_config()
818 void ath11k_ce_get_shadow_config(struct ath11k_base *ab,  in ath11k_ce_get_shadow_config()  argument
821 	if (!ab->hw_params.supports_shadow_regs)  in ath11k_ce_get_shadow_config()
824 	ath11k_hal_srng_get_shadow_config(ab, shadow_cfg, shadow_cfg_len);  in ath11k_ce_get_shadow_config()
834 	ath11k_hal_srng_shadow_config(ab);  in ath11k_ce_get_shadow_config()
835 	ath11k_ce_shadow_config(ab);  in ath11k_ce_get_shadow_config()
838 	ath11k_hal_srng_get_shadow_config(ab, shadow_cfg, shadow_cfg_len);  in ath11k_ce_get_shadow_config()
842 void ath11k_ce_cleanup_pipes(struct ath11k_base *ab)  in ath11k_ce_cleanup_pipes()  argument
847 	ath11k_ce_stop_shadow_timers(ab);  in ath11k_ce_cleanup_pipes()
849 	for (pipe_num = 0; pipe_num < ab->hw_params.ce_count; pipe_num++) {  in ath11k_ce_cleanup_pipes()
850 		pipe = &ab->ce.ce_pipe[pipe_num];  in ath11k_ce_cleanup_pipes()
854 		ath11k_ce_poll_send_completed(ab, pipe_num);  in ath11k_ce_cleanup_pipes()
861 void ath11k_ce_rx_post_buf(struct ath11k_base *ab)  in ath11k_ce_rx_post_buf()  argument
867 	for (i = 0; i < ab->hw_params.ce_count; i++) {  in ath11k_ce_rx_post_buf()
868 		pipe = &ab->ce.ce_pipe[i];  in ath11k_ce_rx_post_buf()
874 			ath11k_warn(ab, "failed to post rx buf to pipe: %d err: %d\n",  in ath11k_ce_rx_post_buf()
876 			mod_timer(&ab->rx_replenish_retry,  in ath11k_ce_rx_post_buf()
887 	struct ath11k_base *ab = from_timer(ab, t, rx_replenish_retry);  in ath11k_ce_rx_replenish_retry()  local
889 	ath11k_ce_rx_post_buf(ab);  in ath11k_ce_rx_replenish_retry()
892 int ath11k_ce_init_pipes(struct ath11k_base *ab)  in ath11k_ce_init_pipes()  argument
898 	ath11k_ce_get_shadow_config(ab, &ab->qmi.ce_cfg.shadow_reg_v2,  in ath11k_ce_init_pipes()
899 				    &ab->qmi.ce_cfg.shadow_reg_v2_len);  in ath11k_ce_init_pipes()
901 	for (i = 0; i < ab->hw_params.ce_count; i++) {  in ath11k_ce_init_pipes()
902 		pipe = &ab->ce.ce_pipe[i];  in ath11k_ce_init_pipes()
905 			ret = ath11k_ce_init_ring(ab, pipe->src_ring, i,  in ath11k_ce_init_pipes()
908 				ath11k_warn(ab, "failed to init src ring: %d\n",  in ath11k_ce_init_pipes()
919 			ret = ath11k_ce_init_ring(ab, pipe->dest_ring, i,  in ath11k_ce_init_pipes()
922 				ath11k_warn(ab, "failed to init dest ring: %d\n",  in ath11k_ce_init_pipes()
936 			ret = ath11k_ce_init_ring(ab, pipe->status_ring, i,  in ath11k_ce_init_pipes()
939 				ath11k_warn(ab, "failed to init dest status ing: %d\n",  in ath11k_ce_init_pipes()
953 void ath11k_ce_free_pipes(struct ath11k_base *ab)  in ath11k_ce_free_pipes()  argument
959 	for (i = 0; i < ab->hw_params.ce_count; i++) {  in ath11k_ce_free_pipes()
960 		pipe = &ab->ce.ce_pipe[i];  in ath11k_ce_free_pipes()
963 			ath11k_dp_shadow_stop_timer(ab, &ab->ce.hp_timer[i]);  in ath11k_ce_free_pipes()
967 			dma_free_coherent(ab->dev,  in ath11k_ce_free_pipes()
978 			dma_free_coherent(ab->dev,  in ath11k_ce_free_pipes()
990 			dma_free_coherent(ab->dev,  in ath11k_ce_free_pipes()
1002 int ath11k_ce_alloc_pipes(struct ath11k_base *ab)  in ath11k_ce_alloc_pipes()  argument
1009 	spin_lock_init(&ab->ce.ce_lock);  in ath11k_ce_alloc_pipes()
1011 	for (i = 0; i < ab->hw_params.ce_count; i++) {  in ath11k_ce_alloc_pipes()
1012 		attr = &ab->hw_params.host_ce_config[i];  in ath11k_ce_alloc_pipes()
1013 		pipe = &ab->ce.ce_pipe[i];  in ath11k_ce_alloc_pipes()
1015 		pipe->ab = ab;  in ath11k_ce_alloc_pipes()
1018 		ret = ath11k_ce_alloc_pipe(ab, i);  in ath11k_ce_alloc_pipes()
1021 			ath11k_ce_free_pipes(ab);  in ath11k_ce_alloc_pipes()
1049 int ath11k_ce_get_attr_flags(struct ath11k_base *ab, int ce_id)  in ath11k_ce_get_attr_flags()  argument
1051 	if (ce_id >= ab->hw_params.ce_count)  in ath11k_ce_get_attr_flags()
1054 	return ab->hw_params.host_ce_config[ce_id].flags;  in ath11k_ce_get_attr_flags()