95 			    struct ixgbe_ring *rx_ring,  in ixgbe_run_xdp_zc()  argument
104 	xdp_prog = READ_ONCE(rx_ring->xdp_prog);  in ixgbe_run_xdp_zc()
119 		err = xdp_do_redirect(rx_ring->netdev, xdp, xdp_prog);  in ixgbe_run_xdp_zc()
126 		trace_xdp_exception(rx_ring->netdev, xdp_prog, act);  in ixgbe_run_xdp_zc()
136 bool ixgbe_alloc_rx_buffers_zc(struct ixgbe_ring *rx_ring, u16 count)  in ixgbe_alloc_rx_buffers_zc()  argument
140 	u16 i = rx_ring->next_to_use;  in ixgbe_alloc_rx_buffers_zc()
148 	rx_desc = IXGBE_RX_DESC(rx_ring, i);  in ixgbe_alloc_rx_buffers_zc()
149 	bi = &rx_ring->rx_buffer_info[i];  in ixgbe_alloc_rx_buffers_zc()
150 	i -= rx_ring->count;  in ixgbe_alloc_rx_buffers_zc()
153 		bi->xdp = xsk_buff_alloc(rx_ring->xsk_pool);  in ixgbe_alloc_rx_buffers_zc()
170 			rx_desc = IXGBE_RX_DESC(rx_ring, 0);  in ixgbe_alloc_rx_buffers_zc()
171 			bi = rx_ring->rx_buffer_info;  in ixgbe_alloc_rx_buffers_zc()
172 			i -= rx_ring->count;  in ixgbe_alloc_rx_buffers_zc()
181 	i += rx_ring->count;  in ixgbe_alloc_rx_buffers_zc()
183 	if (rx_ring->next_to_use != i) {  in ixgbe_alloc_rx_buffers_zc()
184 		rx_ring->next_to_use = i;  in ixgbe_alloc_rx_buffers_zc()
192 		writel(i, rx_ring->tail);  in ixgbe_alloc_rx_buffers_zc()
198 static struct sk_buff *ixgbe_construct_skb_zc(struct ixgbe_ring *rx_ring,  in ixgbe_construct_skb_zc()  argument
206 	skb = __napi_alloc_skb(&rx_ring->q_vector->napi,  in ixgbe_construct_skb_zc()
222 static void ixgbe_inc_ntc(struct ixgbe_ring *rx_ring)  in ixgbe_inc_ntc()  argument
224 	u32 ntc = rx_ring->next_to_clean + 1;  in ixgbe_inc_ntc()
226 	ntc = (ntc < rx_ring->count) ? ntc : 0;  in ixgbe_inc_ntc()
227 	rx_ring->next_to_clean = ntc;  in ixgbe_inc_ntc()
228 	prefetch(IXGBE_RX_DESC(rx_ring, ntc));  in ixgbe_inc_ntc()
232 			  struct ixgbe_ring *rx_ring,  in ixgbe_clean_rx_irq_zc()  argument
237 	u16 cleaned_count = ixgbe_desc_unused(rx_ring);  in ixgbe_clean_rx_irq_zc()
250 				  !ixgbe_alloc_rx_buffers_zc(rx_ring,  in ixgbe_clean_rx_irq_zc()
255 		rx_desc = IXGBE_RX_DESC(rx_ring, rx_ring->next_to_clean);  in ixgbe_clean_rx_irq_zc()
266 		bi = &rx_ring->rx_buffer_info[rx_ring->next_to_clean];  in ixgbe_clean_rx_irq_zc()
274 			ixgbe_inc_ntc(rx_ring);  in ixgbe_clean_rx_irq_zc()
276 			       &rx_ring->rx_buffer_info[rx_ring->next_to_clean];  in ixgbe_clean_rx_irq_zc()
285 			ixgbe_inc_ntc(rx_ring);  in ixgbe_clean_rx_irq_zc()
290 		xsk_buff_dma_sync_for_cpu(bi->xdp, rx_ring->xsk_pool);  in ixgbe_clean_rx_irq_zc()
291 		xdp_res = ixgbe_run_xdp_zc(adapter, rx_ring, bi->xdp);  in ixgbe_clean_rx_irq_zc()
304 			ixgbe_inc_ntc(rx_ring);  in ixgbe_clean_rx_irq_zc()
309 		skb = ixgbe_construct_skb_zc(rx_ring, bi);  in ixgbe_clean_rx_irq_zc()
311 			rx_ring->rx_stats.alloc_rx_buff_failed++;  in ixgbe_clean_rx_irq_zc()
316 		ixgbe_inc_ntc(rx_ring);  in ixgbe_clean_rx_irq_zc()
324 		ixgbe_process_skb_fields(rx_ring, rx_desc, skb);  in ixgbe_clean_rx_irq_zc()
341 	u64_stats_update_begin(&rx_ring->syncp);  in ixgbe_clean_rx_irq_zc()
342 	rx_ring->stats.packets += total_rx_packets;  in ixgbe_clean_rx_irq_zc()
343 	rx_ring->stats.bytes += total_rx_bytes;  in ixgbe_clean_rx_irq_zc()
344 	u64_stats_update_end(&rx_ring->syncp);  in ixgbe_clean_rx_irq_zc()
348 	if (xsk_uses_need_wakeup(rx_ring->xsk_pool)) {  in ixgbe_clean_rx_irq_zc()
349 		if (failure || rx_ring->next_to_clean == rx_ring->next_to_use)  in ixgbe_clean_rx_irq_zc()
350 			xsk_set_rx_need_wakeup(rx_ring->xsk_pool);  in ixgbe_clean_rx_irq_zc()
352 			xsk_clear_rx_need_wakeup(rx_ring->xsk_pool);  in ixgbe_clean_rx_irq_zc()
359 void ixgbe_xsk_clean_rx_ring(struct ixgbe_ring *rx_ring)  in ixgbe_xsk_clean_rx_ring()  argument
364 	for (i = 0; i < rx_ring->count; i++) {  in ixgbe_xsk_clean_rx_ring()
365 		bi = &rx_ring->rx_buffer_info[i];  in ixgbe_xsk_clean_rx_ring()