41 static struct page *efx_reuse_page(struct efx_rx_queue *rx_queue)  in efx_reuse_page()  argument
43 	struct efx_nic *efx = rx_queue->efx;  in efx_reuse_page()
48 	index = rx_queue->page_remove & rx_queue->page_ptr_mask;  in efx_reuse_page()
49 	page = rx_queue->page_ring[index];  in efx_reuse_page()
53 	rx_queue->page_ring[index] = NULL;  in efx_reuse_page()
55 	if (rx_queue->page_remove != rx_queue->page_add)  in efx_reuse_page()
56 		++rx_queue->page_remove;  in efx_reuse_page()
60 		++rx_queue->page_recycle_count;  in efx_reuse_page()
68 		++rx_queue->page_recycle_failed;  in efx_reuse_page()
81 	struct efx_rx_queue *rx_queue = efx_channel_get_rx_queue(channel);  in efx_recycle_rx_page()  local
82 	struct efx_nic *efx = rx_queue->efx;  in efx_recycle_rx_page()
90 	index = rx_queue->page_add & rx_queue->page_ptr_mask;  in efx_recycle_rx_page()
91 	if (rx_queue->page_ring[index] == NULL) {  in efx_recycle_rx_page()
92 		unsigned int read_index = rx_queue->page_remove &  in efx_recycle_rx_page()
93 			rx_queue->page_ptr_mask;  in efx_recycle_rx_page()
100 			++rx_queue->page_remove;  in efx_recycle_rx_page()
101 		rx_queue->page_ring[index] = page;  in efx_recycle_rx_page()
102 		++rx_queue->page_add;  in efx_recycle_rx_page()
105 	++rx_queue->page_recycle_full;  in efx_recycle_rx_page()
115 	struct efx_rx_queue *rx_queue = efx_channel_get_rx_queue(channel);  in efx_recycle_rx_pages()  local
119 		rx_buf = efx_rx_buf_next(rx_queue, rx_buf);  in efx_recycle_rx_pages()
127 	struct efx_rx_queue *rx_queue = efx_channel_get_rx_queue(channel);  in efx_discard_rx_packet()  local
131 	efx_free_rx_buffers(rx_queue, rx_buf, n_frags);  in efx_discard_rx_packet()
134 static void efx_init_rx_recycle_ring(struct efx_rx_queue *rx_queue)  in efx_init_rx_recycle_ring()  argument
137 	struct efx_nic *efx = rx_queue->efx;  in efx_init_rx_recycle_ring()
151 	rx_queue->page_ring = kcalloc(page_ring_size,  in efx_init_rx_recycle_ring()
152 				      sizeof(*rx_queue->page_ring), GFP_KERNEL);  in efx_init_rx_recycle_ring()
153 	rx_queue->page_ptr_mask = page_ring_size - 1;  in efx_init_rx_recycle_ring()
156 static void efx_fini_rx_recycle_ring(struct efx_rx_queue *rx_queue)  in efx_fini_rx_recycle_ring()  argument
158 	struct efx_nic *efx = rx_queue->efx;  in efx_fini_rx_recycle_ring()
162 	for (i = 0; i <= rx_queue->page_ptr_mask; i++) {  in efx_fini_rx_recycle_ring()
163 		struct page *page = rx_queue->page_ring[i];  in efx_fini_rx_recycle_ring()
175 	kfree(rx_queue->page_ring);  in efx_fini_rx_recycle_ring()
176 	rx_queue->page_ring = NULL;  in efx_fini_rx_recycle_ring()
179 static void efx_fini_rx_buffer(struct efx_rx_queue *rx_queue,  in efx_fini_rx_buffer()  argument
188 		efx_unmap_rx_buffer(rx_queue->efx, rx_buf);  in efx_fini_rx_buffer()
189 		efx_free_rx_buffers(rx_queue, rx_buf, 1);  in efx_fini_rx_buffer()
194 int efx_probe_rx_queue(struct efx_rx_queue *rx_queue)  in efx_probe_rx_queue()  argument
196 	struct efx_nic *efx = rx_queue->efx;  in efx_probe_rx_queue()
203 	rx_queue->ptr_mask = entries - 1;  in efx_probe_rx_queue()
207 		  efx_rx_queue_index(rx_queue), efx->rxq_entries,  in efx_probe_rx_queue()
208 		  rx_queue->ptr_mask);  in efx_probe_rx_queue()
211 	rx_queue->buffer = kcalloc(entries, sizeof(*rx_queue->buffer),  in efx_probe_rx_queue()
213 	if (!rx_queue->buffer)  in efx_probe_rx_queue()
216 	rc = efx_nic_probe_rx(rx_queue);  in efx_probe_rx_queue()
218 		kfree(rx_queue->buffer);  in efx_probe_rx_queue()
219 		rx_queue->buffer = NULL;  in efx_probe_rx_queue()
225 void efx_init_rx_queue(struct efx_rx_queue *rx_queue)  in efx_init_rx_queue()  argument
228 	struct efx_nic *efx = rx_queue->efx;  in efx_init_rx_queue()
231 	netif_dbg(rx_queue->efx, drv, rx_queue->efx->net_dev,  in efx_init_rx_queue()
232 		  "initialising RX queue %d\n", efx_rx_queue_index(rx_queue));  in efx_init_rx_queue()
235 	rx_queue->added_count = 0;  in efx_init_rx_queue()
236 	rx_queue->notified_count = 0;  in efx_init_rx_queue()
237 	rx_queue->removed_count = 0;  in efx_init_rx_queue()
238 	rx_queue->min_fill = -1U;  in efx_init_rx_queue()
239 	efx_init_rx_recycle_ring(rx_queue);  in efx_init_rx_queue()
241 	rx_queue->page_remove = 0;  in efx_init_rx_queue()
242 	rx_queue->page_add = rx_queue->page_ptr_mask + 1;  in efx_init_rx_queue()
243 	rx_queue->page_recycle_count = 0;  in efx_init_rx_queue()
244 	rx_queue->page_recycle_failed = 0;  in efx_init_rx_queue()
245 	rx_queue->page_recycle_full = 0;  in efx_init_rx_queue()
259 	rx_queue->max_fill = max_fill;  in efx_init_rx_queue()
260 	rx_queue->fast_fill_trigger = trigger;  in efx_init_rx_queue()
261 	rx_queue->refill_enabled = true;  in efx_init_rx_queue()
264 	rc = xdp_rxq_info_reg(&rx_queue->xdp_rxq_info, efx->net_dev,  in efx_init_rx_queue()
265 			      rx_queue->core_index);  in efx_init_rx_queue()
273 		rx_queue->xdp_rxq_info_valid = true;  in efx_init_rx_queue()
277 	efx_nic_init_rx(rx_queue);  in efx_init_rx_queue()
280 void efx_fini_rx_queue(struct efx_rx_queue *rx_queue)  in efx_fini_rx_queue()  argument
285 	netif_dbg(rx_queue->efx, drv, rx_queue->efx->net_dev,  in efx_fini_rx_queue()
286 		  "shutting down RX queue %d\n", efx_rx_queue_index(rx_queue));  in efx_fini_rx_queue()
288 	del_timer_sync(&rx_queue->slow_fill);  in efx_fini_rx_queue()
291 	if (rx_queue->buffer) {  in efx_fini_rx_queue()
292 		for (i = rx_queue->removed_count; i < rx_queue->added_count;  in efx_fini_rx_queue()
294 			unsigned int index = i & rx_queue->ptr_mask;  in efx_fini_rx_queue()
296 			rx_buf = efx_rx_buffer(rx_queue, index);  in efx_fini_rx_queue()
297 			efx_fini_rx_buffer(rx_queue, rx_buf);  in efx_fini_rx_queue()
301 	efx_fini_rx_recycle_ring(rx_queue);  in efx_fini_rx_queue()
303 	if (rx_queue->xdp_rxq_info_valid)  in efx_fini_rx_queue()
304 		xdp_rxq_info_unreg(&rx_queue->xdp_rxq_info);  in efx_fini_rx_queue()
306 	rx_queue->xdp_rxq_info_valid = false;  in efx_fini_rx_queue()
309 void efx_remove_rx_queue(struct efx_rx_queue *rx_queue)  in efx_remove_rx_queue()  argument
311 	netif_dbg(rx_queue->efx, drv, rx_queue->efx->net_dev,  in efx_remove_rx_queue()
312 		  "destroying RX queue %d\n", efx_rx_queue_index(rx_queue));  in efx_remove_rx_queue()
314 	efx_nic_remove_rx(rx_queue);  in efx_remove_rx_queue()
316 	kfree(rx_queue->buffer);  in efx_remove_rx_queue()
317 	rx_queue->buffer = NULL;  in efx_remove_rx_queue()
338 void efx_free_rx_buffers(struct efx_rx_queue *rx_queue,  in efx_free_rx_buffers()  argument
347 		rx_buf = efx_rx_buf_next(rx_queue, rx_buf);  in efx_free_rx_buffers()
353 	struct efx_rx_queue *rx_queue = from_timer(rx_queue, t, slow_fill);  in efx_rx_slow_fill()  local
356 	efx_nic_generate_fill_event(rx_queue);  in efx_rx_slow_fill()
357 	++rx_queue->slow_fill_count;  in efx_rx_slow_fill()
360 void efx_schedule_slow_fill(struct efx_rx_queue *rx_queue)  in efx_schedule_slow_fill()  argument
362 	mod_timer(&rx_queue->slow_fill, jiffies + msecs_to_jiffies(10));  in efx_schedule_slow_fill()
374 static int efx_init_rx_buffers(struct efx_rx_queue *rx_queue, bool atomic)  in efx_init_rx_buffers()  argument
377 	struct efx_nic *efx = rx_queue->efx;  in efx_init_rx_buffers()
385 		page = efx_reuse_page(rx_queue);  in efx_init_rx_buffers()
412 			index = rx_queue->added_count & rx_queue->ptr_mask;  in efx_init_rx_buffers()
413 			rx_buf = efx_rx_buffer(rx_queue, index);  in efx_init_rx_buffers()
421 			++rx_queue->added_count;  in efx_init_rx_buffers()
458 void efx_fast_push_rx_descriptors(struct efx_rx_queue *rx_queue, bool atomic)  in efx_fast_push_rx_descriptors()  argument
460 	struct efx_nic *efx = rx_queue->efx;  in efx_fast_push_rx_descriptors()
464 	if (!rx_queue->refill_enabled)  in efx_fast_push_rx_descriptors()
468 	fill_level = (rx_queue->added_count - rx_queue->removed_count);  in efx_fast_push_rx_descriptors()
469 	EFX_WARN_ON_ONCE_PARANOID(fill_level > rx_queue->efx->rxq_entries);  in efx_fast_push_rx_descriptors()
470 	if (fill_level >= rx_queue->fast_fill_trigger)  in efx_fast_push_rx_descriptors()
474 	if (unlikely(fill_level < rx_queue->min_fill)) {  in efx_fast_push_rx_descriptors()
476 			rx_queue->min_fill = fill_level;  in efx_fast_push_rx_descriptors()
480 	space = rx_queue->max_fill - fill_level;  in efx_fast_push_rx_descriptors()
483 	netif_vdbg(rx_queue->efx, rx_status, rx_queue->efx->net_dev,  in efx_fast_push_rx_descriptors()
486 		   efx_rx_queue_index(rx_queue), fill_level,  in efx_fast_push_rx_descriptors()
487 		   rx_queue->max_fill);  in efx_fast_push_rx_descriptors()
490 		rc = efx_init_rx_buffers(rx_queue, atomic);  in efx_fast_push_rx_descriptors()
493 			efx_schedule_slow_fill(rx_queue);  in efx_fast_push_rx_descriptors()
498 	netif_vdbg(rx_queue->efx, rx_status, rx_queue->efx->net_dev,  in efx_fast_push_rx_descriptors()
500 		   "to level %d\n", efx_rx_queue_index(rx_queue),  in efx_fast_push_rx_descriptors()
501 		   rx_queue->added_count - rx_queue->removed_count);  in efx_fast_push_rx_descriptors()
504 	if (rx_queue->notified_count != rx_queue->added_count)  in efx_fast_push_rx_descriptors()
505 		efx_nic_notify_rx_desc(rx_queue);  in efx_fast_push_rx_descriptors()
521 		struct efx_rx_queue *rx_queue;  in efx_rx_packet_gro()  local
523 		rx_queue = efx_channel_get_rx_queue(channel);  in efx_rx_packet_gro()
524 		efx_free_rx_buffers(rx_queue, rx_buf, n_frags);  in efx_rx_packet_gro()
550 		rx_buf = efx_rx_buf_next(&channel->rx_queue, rx_buf);  in efx_rx_packet_gro()
556 	skb_record_rx_queue(skb, channel->rx_queue.core_index);  in efx_rx_packet_gro()