13 int i40e_alloc_rx_bi_zc(struct i40e_ring *rx_ring)  in i40e_alloc_rx_bi_zc()  argument
15 	unsigned long sz = sizeof(*rx_ring->rx_bi_zc) * rx_ring->count;  in i40e_alloc_rx_bi_zc()
17 	rx_ring->rx_bi_zc = kzalloc(sz, GFP_KERNEL);  in i40e_alloc_rx_bi_zc()
18 	return rx_ring->rx_bi_zc ? 0 : -ENOMEM;  in i40e_alloc_rx_bi_zc()
21 void i40e_clear_rx_bi_zc(struct i40e_ring *rx_ring)  in i40e_clear_rx_bi_zc()  argument
23 	memset(rx_ring->rx_bi_zc, 0,  in i40e_clear_rx_bi_zc()
24 	       sizeof(*rx_ring->rx_bi_zc) * rx_ring->count);  in i40e_clear_rx_bi_zc()
27 static struct xdp_buff **i40e_rx_bi(struct i40e_ring *rx_ring, u32 idx)  in i40e_rx_bi()  argument
29 	return &rx_ring->rx_bi_zc[idx];  in i40e_rx_bi()
149 static int i40e_run_xdp_zc(struct i40e_ring *rx_ring, struct xdp_buff *xdp)  in i40e_run_xdp_zc()  argument
159 	xdp_prog = READ_ONCE(rx_ring->xdp_prog);  in i40e_run_xdp_zc()
163 		err = xdp_do_redirect(rx_ring->netdev, xdp, xdp_prog);  in i40e_run_xdp_zc()
173 		xdp_ring = rx_ring->vsi->xdp_rings[rx_ring->queue_index];  in i40e_run_xdp_zc()
183 		trace_xdp_exception(rx_ring->netdev, xdp_prog, act);  in i40e_run_xdp_zc()
192 bool i40e_alloc_rx_buffers_zc(struct i40e_ring *rx_ring, u16 count)  in i40e_alloc_rx_buffers_zc()  argument
194 	u16 ntu = rx_ring->next_to_use;  in i40e_alloc_rx_buffers_zc()
200 	rx_desc = I40E_RX_DESC(rx_ring, ntu);  in i40e_alloc_rx_buffers_zc()
201 	bi = i40e_rx_bi(rx_ring, ntu);  in i40e_alloc_rx_buffers_zc()
203 		xdp = xsk_buff_alloc(rx_ring->xsk_pool);  in i40e_alloc_rx_buffers_zc()
217 		if (unlikely(ntu == rx_ring->count)) {  in i40e_alloc_rx_buffers_zc()
218 			rx_desc = I40E_RX_DESC(rx_ring, 0);  in i40e_alloc_rx_buffers_zc()
219 			bi = i40e_rx_bi(rx_ring, 0);  in i40e_alloc_rx_buffers_zc()
225 	if (rx_ring->next_to_use != ntu) {  in i40e_alloc_rx_buffers_zc()
228 		i40e_release_rx_desc(rx_ring, ntu);  in i40e_alloc_rx_buffers_zc()
243 static struct sk_buff *i40e_construct_skb_zc(struct i40e_ring *rx_ring,  in i40e_construct_skb_zc()  argument
251 	skb = __napi_alloc_skb(&rx_ring->q_vector->napi,  in i40e_construct_skb_zc()
267 static void i40e_handle_xdp_result_zc(struct i40e_ring *rx_ring,  in i40e_handle_xdp_result_zc()  argument
294 		skb = i40e_construct_skb_zc(rx_ring, xdp_buff);  in i40e_handle_xdp_result_zc()
296 			rx_ring->rx_stats.alloc_buff_failed++;  in i40e_handle_xdp_result_zc()
309 		i40e_process_skb_fields(rx_ring, rx_desc, skb);  in i40e_handle_xdp_result_zc()
310 		napi_gro_receive(&rx_ring->q_vector->napi, skb);  in i40e_handle_xdp_result_zc()
326 int i40e_clean_rx_irq_zc(struct i40e_ring *rx_ring, int budget)  in i40e_clean_rx_irq_zc()  argument
329 	u16 cleaned_count = I40E_DESC_UNUSED(rx_ring);  in i40e_clean_rx_irq_zc()
330 	u16 next_to_clean = rx_ring->next_to_clean;  in i40e_clean_rx_irq_zc()
331 	u16 count_mask = rx_ring->count - 1;  in i40e_clean_rx_irq_zc()
343 		rx_desc = I40E_RX_DESC(rx_ring, next_to_clean);  in i40e_clean_rx_irq_zc()
353 			i40e_clean_programming_status(rx_ring,  in i40e_clean_rx_irq_zc()
356 			bi = *i40e_rx_bi(rx_ring, next_to_clean);  in i40e_clean_rx_irq_zc()
367 		bi = *i40e_rx_bi(rx_ring, next_to_clean);  in i40e_clean_rx_irq_zc()
369 		xsk_buff_dma_sync_for_cpu(bi, rx_ring->xsk_pool);  in i40e_clean_rx_irq_zc()
371 		xdp_res = i40e_run_xdp_zc(rx_ring, bi);  in i40e_clean_rx_irq_zc()
372 		i40e_handle_xdp_result_zc(rx_ring, bi, rx_desc, &rx_packets,  in i40e_clean_rx_irq_zc()
380 	rx_ring->next_to_clean = next_to_clean;  in i40e_clean_rx_irq_zc()
381 	cleaned_count = (next_to_clean - rx_ring->next_to_use - 1) & count_mask;  in i40e_clean_rx_irq_zc()
384 		failure = !i40e_alloc_rx_buffers_zc(rx_ring, cleaned_count);  in i40e_clean_rx_irq_zc()
386 	i40e_finalize_xdp_rx(rx_ring, xdp_xmit);  in i40e_clean_rx_irq_zc()
387 	i40e_update_rx_stats(rx_ring, total_rx_bytes, total_rx_packets);  in i40e_clean_rx_irq_zc()
389 	if (xsk_uses_need_wakeup(rx_ring->xsk_pool)) {  in i40e_clean_rx_irq_zc()
390 		if (failure || next_to_clean == rx_ring->next_to_use)  in i40e_clean_rx_irq_zc()
391 			xsk_set_rx_need_wakeup(rx_ring->xsk_pool);  in i40e_clean_rx_irq_zc()
393 			xsk_clear_rx_need_wakeup(rx_ring->xsk_pool);  in i40e_clean_rx_irq_zc()
618 void i40e_xsk_clean_rx_ring(struct i40e_ring *rx_ring)  in i40e_xsk_clean_rx_ring()  argument
620 	u16 count_mask = rx_ring->count - 1;  in i40e_xsk_clean_rx_ring()
621 	u16 ntc = rx_ring->next_to_clean;  in i40e_xsk_clean_rx_ring()
622 	u16 ntu = rx_ring->next_to_use;  in i40e_xsk_clean_rx_ring()
625 		struct xdp_buff *rx_bi = *i40e_rx_bi(rx_ring, ntc);  in i40e_xsk_clean_rx_ring()