12 int i40e_alloc_rx_bi_zc(struct i40e_ring *rx_ring)  in i40e_alloc_rx_bi_zc()  argument
14 	unsigned long sz = sizeof(*rx_ring->rx_bi_zc) * rx_ring->count;  in i40e_alloc_rx_bi_zc()
16 	rx_ring->rx_bi_zc = kzalloc(sz, GFP_KERNEL);  in i40e_alloc_rx_bi_zc()
17 	return rx_ring->rx_bi_zc ? 0 : -ENOMEM;  in i40e_alloc_rx_bi_zc()
20 void i40e_clear_rx_bi_zc(struct i40e_ring *rx_ring)  in i40e_clear_rx_bi_zc()  argument
22 	memset(rx_ring->rx_bi_zc, 0,  in i40e_clear_rx_bi_zc()
23 	       sizeof(*rx_ring->rx_bi_zc) * rx_ring->count);  in i40e_clear_rx_bi_zc()
26 static struct xdp_buff **i40e_rx_bi(struct i40e_ring *rx_ring, u32 idx)  in i40e_rx_bi()  argument
28 	return &rx_ring->rx_bi_zc[idx];  in i40e_rx_bi()
148 static int i40e_run_xdp_zc(struct i40e_ring *rx_ring, struct xdp_buff *xdp)  in i40e_run_xdp_zc()  argument
159 	xdp_prog = READ_ONCE(rx_ring->xdp_prog);  in i40e_run_xdp_zc()
166 		xdp_ring = rx_ring->vsi->xdp_rings[rx_ring->queue_index];  in i40e_run_xdp_zc()
170 		err = xdp_do_redirect(rx_ring->netdev, xdp, xdp_prog);  in i40e_run_xdp_zc()
177 		trace_xdp_exception(rx_ring->netdev, xdp_prog, act);  in i40e_run_xdp_zc()
187 bool i40e_alloc_rx_buffers_zc(struct i40e_ring *rx_ring, u16 count)  in i40e_alloc_rx_buffers_zc()  argument
189 	u16 ntu = rx_ring->next_to_use;  in i40e_alloc_rx_buffers_zc()
195 	rx_desc = I40E_RX_DESC(rx_ring, ntu);  in i40e_alloc_rx_buffers_zc()
196 	bi = i40e_rx_bi(rx_ring, ntu);  in i40e_alloc_rx_buffers_zc()
198 		xdp = xsk_buff_alloc(rx_ring->xsk_pool);  in i40e_alloc_rx_buffers_zc()
212 		if (unlikely(ntu == rx_ring->count)) {  in i40e_alloc_rx_buffers_zc()
213 			rx_desc = I40E_RX_DESC(rx_ring, 0);  in i40e_alloc_rx_buffers_zc()
214 			bi = i40e_rx_bi(rx_ring, 0);  in i40e_alloc_rx_buffers_zc()
222 	if (rx_ring->next_to_use != ntu)  in i40e_alloc_rx_buffers_zc()
223 		i40e_release_rx_desc(rx_ring, ntu);  in i40e_alloc_rx_buffers_zc()
237 static struct sk_buff *i40e_construct_skb_zc(struct i40e_ring *rx_ring,  in i40e_construct_skb_zc()  argument
245 	skb = __napi_alloc_skb(&rx_ring->q_vector->napi,  in i40e_construct_skb_zc()
264 static void i40e_inc_ntc(struct i40e_ring *rx_ring)  in i40e_inc_ntc()  argument
266 	u32 ntc = rx_ring->next_to_clean + 1;  in i40e_inc_ntc()
268 	ntc = (ntc < rx_ring->count) ? ntc : 0;  in i40e_inc_ntc()
269 	rx_ring->next_to_clean = ntc;  in i40e_inc_ntc()
279 int i40e_clean_rx_irq_zc(struct i40e_ring *rx_ring, int budget)  in i40e_clean_rx_irq_zc()  argument
282 	u16 cleaned_count = I40E_DESC_UNUSED(rx_ring);  in i40e_clean_rx_irq_zc()
293 		rx_desc = I40E_RX_DESC(rx_ring, rx_ring->next_to_clean);  in i40e_clean_rx_irq_zc()
303 			i40e_clean_programming_status(rx_ring,  in i40e_clean_rx_irq_zc()
306 			bi = i40e_rx_bi(rx_ring, rx_ring->next_to_clean);  in i40e_clean_rx_irq_zc()
310 			i40e_inc_ntc(rx_ring);  in i40e_clean_rx_irq_zc()
314 		bi = i40e_rx_bi(rx_ring, rx_ring->next_to_clean);  in i40e_clean_rx_irq_zc()
320 		bi = i40e_rx_bi(rx_ring, rx_ring->next_to_clean);  in i40e_clean_rx_irq_zc()
322 		xsk_buff_dma_sync_for_cpu(*bi, rx_ring->xsk_pool);  in i40e_clean_rx_irq_zc()
324 		xdp_res = i40e_run_xdp_zc(rx_ring, *bi);  in i40e_clean_rx_irq_zc()
336 			i40e_inc_ntc(rx_ring);  in i40e_clean_rx_irq_zc()
347 		skb = i40e_construct_skb_zc(rx_ring, *bi);  in i40e_clean_rx_irq_zc()
350 			rx_ring->rx_stats.alloc_buff_failed++;  in i40e_clean_rx_irq_zc()
355 		i40e_inc_ntc(rx_ring);  in i40e_clean_rx_irq_zc()
363 		i40e_process_skb_fields(rx_ring, rx_desc, skb);  in i40e_clean_rx_irq_zc()
364 		napi_gro_receive(&rx_ring->q_vector->napi, skb);  in i40e_clean_rx_irq_zc()
368 		failure = !i40e_alloc_rx_buffers_zc(rx_ring, cleaned_count);  in i40e_clean_rx_irq_zc()
370 	i40e_finalize_xdp_rx(rx_ring, xdp_xmit);  in i40e_clean_rx_irq_zc()
371 	i40e_update_rx_stats(rx_ring, total_rx_bytes, total_rx_packets);  in i40e_clean_rx_irq_zc()
373 	if (xsk_uses_need_wakeup(rx_ring->xsk_pool)) {  in i40e_clean_rx_irq_zc()
374 		if (failure || rx_ring->next_to_clean == rx_ring->next_to_use)  in i40e_clean_rx_irq_zc()
375 			xsk_set_rx_need_wakeup(rx_ring->xsk_pool);  in i40e_clean_rx_irq_zc()
377 			xsk_clear_rx_need_wakeup(rx_ring->xsk_pool);  in i40e_clean_rx_irq_zc()
558 void i40e_xsk_clean_rx_ring(struct i40e_ring *rx_ring)  in i40e_xsk_clean_rx_ring()  argument
562 	for (i = 0; i < rx_ring->count; i++) {  in i40e_xsk_clean_rx_ring()
563 		struct xdp_buff *rx_bi = *i40e_rx_bi(rx_ring, i);  in i40e_xsk_clean_rx_ring()