76 ef4_rx_buf_next(struct ef4_rx_queue *rx_queue, struct ef4_rx_buffer *rx_buf)  in ef4_rx_buf_next()  argument
78 	if (unlikely(rx_buf == ef4_rx_buffer(rx_queue, rx_queue->ptr_mask)))  in ef4_rx_buf_next()
79 		return ef4_rx_buffer(rx_queue, 0);  in ef4_rx_buf_next()
106 static struct page *ef4_reuse_page(struct ef4_rx_queue *rx_queue)  in ef4_reuse_page()  argument
108 	struct ef4_nic *efx = rx_queue->efx;  in ef4_reuse_page()
113 	index = rx_queue->page_remove & rx_queue->page_ptr_mask;  in ef4_reuse_page()
114 	page = rx_queue->page_ring[index];  in ef4_reuse_page()
118 	rx_queue->page_ring[index] = NULL;  in ef4_reuse_page()
120 	if (rx_queue->page_remove != rx_queue->page_add)  in ef4_reuse_page()
121 		++rx_queue->page_remove;  in ef4_reuse_page()
125 		++rx_queue->page_recycle_count;  in ef4_reuse_page()
133 		++rx_queue->page_recycle_failed;  in ef4_reuse_page()
150 static int ef4_init_rx_buffers(struct ef4_rx_queue *rx_queue, bool atomic)  in ef4_init_rx_buffers()  argument
152 	struct ef4_nic *efx = rx_queue->efx;  in ef4_init_rx_buffers()
162 		page = ef4_reuse_page(rx_queue);  in ef4_init_rx_buffers()
189 			index = rx_queue->added_count & rx_queue->ptr_mask;  in ef4_init_rx_buffers()
190 			rx_buf = ef4_rx_buffer(rx_queue, index);  in ef4_init_rx_buffers()
196 			++rx_queue->added_count;  in ef4_init_rx_buffers()
225 static void ef4_free_rx_buffers(struct ef4_rx_queue *rx_queue,  in ef4_free_rx_buffers()  argument
234 		rx_buf = ef4_rx_buf_next(rx_queue, rx_buf);  in ef4_free_rx_buffers()
246 	struct ef4_rx_queue *rx_queue = ef4_channel_get_rx_queue(channel);  in ef4_recycle_rx_page()  local
247 	struct ef4_nic *efx = rx_queue->efx;  in ef4_recycle_rx_page()
254 	index = rx_queue->page_add & rx_queue->page_ptr_mask;  in ef4_recycle_rx_page()
255 	if (rx_queue->page_ring[index] == NULL) {  in ef4_recycle_rx_page()
256 		unsigned read_index = rx_queue->page_remove &  in ef4_recycle_rx_page()
257 			rx_queue->page_ptr_mask;  in ef4_recycle_rx_page()
264 			++rx_queue->page_remove;  in ef4_recycle_rx_page()
265 		rx_queue->page_ring[index] = page;  in ef4_recycle_rx_page()
266 		++rx_queue->page_add;  in ef4_recycle_rx_page()
269 	++rx_queue->page_recycle_full;  in ef4_recycle_rx_page()
274 static void ef4_fini_rx_buffer(struct ef4_rx_queue *rx_queue,  in ef4_fini_rx_buffer()  argument
283 		ef4_unmap_rx_buffer(rx_queue->efx, rx_buf);  in ef4_fini_rx_buffer()
284 		ef4_free_rx_buffers(rx_queue, rx_buf, 1);  in ef4_fini_rx_buffer()
294 	struct ef4_rx_queue *rx_queue = ef4_channel_get_rx_queue(channel);  in ef4_recycle_rx_pages()  local
298 		rx_buf = ef4_rx_buf_next(rx_queue, rx_buf);  in ef4_recycle_rx_pages()
306 	struct ef4_rx_queue *rx_queue = ef4_channel_get_rx_queue(channel);  in ef4_discard_rx_packet()  local
310 	ef4_free_rx_buffers(rx_queue, rx_buf, n_frags);  in ef4_discard_rx_packet()
326 void ef4_fast_push_rx_descriptors(struct ef4_rx_queue *rx_queue, bool atomic)  in ef4_fast_push_rx_descriptors()  argument
328 	struct ef4_nic *efx = rx_queue->efx;  in ef4_fast_push_rx_descriptors()
332 	if (!rx_queue->refill_enabled)  in ef4_fast_push_rx_descriptors()
336 	fill_level = (rx_queue->added_count - rx_queue->removed_count);  in ef4_fast_push_rx_descriptors()
337 	EF4_BUG_ON_PARANOID(fill_level > rx_queue->efx->rxq_entries);  in ef4_fast_push_rx_descriptors()
338 	if (fill_level >= rx_queue->fast_fill_trigger)  in ef4_fast_push_rx_descriptors()
342 	if (unlikely(fill_level < rx_queue->min_fill)) {  in ef4_fast_push_rx_descriptors()
344 			rx_queue->min_fill = fill_level;  in ef4_fast_push_rx_descriptors()
348 	space = rx_queue->max_fill - fill_level;  in ef4_fast_push_rx_descriptors()
351 	netif_vdbg(rx_queue->efx, rx_status, rx_queue->efx->net_dev,  in ef4_fast_push_rx_descriptors()
354 		   ef4_rx_queue_index(rx_queue), fill_level,  in ef4_fast_push_rx_descriptors()
355 		   rx_queue->max_fill);  in ef4_fast_push_rx_descriptors()
359 		rc = ef4_init_rx_buffers(rx_queue, atomic);  in ef4_fast_push_rx_descriptors()
362 			if (rx_queue->added_count == rx_queue->removed_count)  in ef4_fast_push_rx_descriptors()
363 				ef4_schedule_slow_fill(rx_queue);  in ef4_fast_push_rx_descriptors()
368 	netif_vdbg(rx_queue->efx, rx_status, rx_queue->efx->net_dev,  in ef4_fast_push_rx_descriptors()
370 		   "to level %d\n", ef4_rx_queue_index(rx_queue),  in ef4_fast_push_rx_descriptors()
371 		   rx_queue->added_count - rx_queue->removed_count);  in ef4_fast_push_rx_descriptors()
374 	if (rx_queue->notified_count != rx_queue->added_count)  in ef4_fast_push_rx_descriptors()
375 		ef4_nic_notify_rx_desc(rx_queue);  in ef4_fast_push_rx_descriptors()
380 	struct ef4_rx_queue *rx_queue = from_timer(rx_queue, t, slow_fill);  in ef4_rx_slow_fill()  local
383 	ef4_nic_generate_fill_event(rx_queue);  in ef4_rx_slow_fill()
384 	++rx_queue->slow_fill_count;  in ef4_rx_slow_fill()
387 static void ef4_rx_packet__check_len(struct ef4_rx_queue *rx_queue,  in ef4_rx_packet__check_len()  argument
391 	struct ef4_nic *efx = rx_queue->efx;  in ef4_rx_packet__check_len()
407 				  ef4_rx_queue_index(rx_queue), len, max_len,  in ef4_rx_packet__check_len()
415 				  ef4_rx_queue_index(rx_queue), len, max_len);  in ef4_rx_packet__check_len()
418 	ef4_rx_queue_channel(rx_queue)->n_rx_overlength++;  in ef4_rx_packet__check_len()
434 		struct ef4_rx_queue *rx_queue;  in ef4_rx_packet_gro()  local
436 		rx_queue = ef4_channel_get_rx_queue(channel);  in ef4_rx_packet_gro()
437 		ef4_free_rx_buffers(rx_queue, rx_buf, n_frags);  in ef4_rx_packet_gro()
456 		rx_buf = ef4_rx_buf_next(&channel->rx_queue, rx_buf);  in ef4_rx_packet_gro()
462 	skb_record_rx_queue(skb, channel->rx_queue.core_index);  in ef4_rx_packet_gro()
507 			rx_buf = ef4_rx_buf_next(&channel->rx_queue, rx_buf);  in ef4_rx_mk_skb()
525 void ef4_rx_packet(struct ef4_rx_queue *rx_queue, unsigned int index,  in ef4_rx_packet()  argument
528 	struct ef4_nic *efx = rx_queue->efx;  in ef4_rx_packet()
529 	struct ef4_channel *channel = ef4_rx_queue_channel(rx_queue);  in ef4_rx_packet()
532 	rx_queue->rx_packets++;  in ef4_rx_packet()
534 	rx_buf = ef4_rx_buffer(rx_queue, index);  in ef4_rx_packet()
540 			ef4_rx_packet__check_len(rx_queue, rx_buf, len);  in ef4_rx_packet()
554 		   ef4_rx_queue_index(rx_queue), index,  in ef4_rx_packet()
555 		   (index + n_frags - 1) & rx_queue->ptr_mask, len,  in ef4_rx_packet()
591 			rx_buf = ef4_rx_buf_next(rx_queue, rx_buf);  in ef4_rx_packet()
601 	rx_buf = ef4_rx_buffer(rx_queue, index);  in ef4_rx_packet()
621 		struct ef4_rx_queue *rx_queue;  in ef4_rx_deliver()  local
623 		rx_queue = ef4_channel_get_rx_queue(channel);  in ef4_rx_deliver()
624 		ef4_free_rx_buffers(rx_queue, rx_buf, n_frags);  in ef4_rx_deliver()
627 	skb_record_rx_queue(skb, channel->rx_queue.core_index);  in ef4_rx_deliver()
647 		ef4_rx_buffer(&channel->rx_queue, channel->rx_pkt_index);  in __ef4_rx_packet()
661 		struct ef4_rx_queue *rx_queue;  in __ef4_rx_packet()  local
664 		rx_queue = ef4_channel_get_rx_queue(channel);  in __ef4_rx_packet()
665 		ef4_free_rx_buffers(rx_queue, rx_buf,  in __ef4_rx_packet()
681 int ef4_probe_rx_queue(struct ef4_rx_queue *rx_queue)  in ef4_probe_rx_queue()  argument
683 	struct ef4_nic *efx = rx_queue->efx;  in ef4_probe_rx_queue()
690 	rx_queue->ptr_mask = entries - 1;  in ef4_probe_rx_queue()
694 		  ef4_rx_queue_index(rx_queue), efx->rxq_entries,  in ef4_probe_rx_queue()
695 		  rx_queue->ptr_mask);  in ef4_probe_rx_queue()
698 	rx_queue->buffer = kcalloc(entries, sizeof(*rx_queue->buffer),  in ef4_probe_rx_queue()
700 	if (!rx_queue->buffer)  in ef4_probe_rx_queue()
703 	rc = ef4_nic_probe_rx(rx_queue);  in ef4_probe_rx_queue()
705 		kfree(rx_queue->buffer);  in ef4_probe_rx_queue()
706 		rx_queue->buffer = NULL;  in ef4_probe_rx_queue()
713 				     struct ef4_rx_queue *rx_queue)  in ef4_init_rx_recycle_ring()  argument
729 	rx_queue->page_ring = kcalloc(page_ring_size,  in ef4_init_rx_recycle_ring()
730 				      sizeof(*rx_queue->page_ring), GFP_KERNEL);  in ef4_init_rx_recycle_ring()
731 	rx_queue->page_ptr_mask = page_ring_size - 1;  in ef4_init_rx_recycle_ring()
734 void ef4_init_rx_queue(struct ef4_rx_queue *rx_queue)  in ef4_init_rx_queue()  argument
736 	struct ef4_nic *efx = rx_queue->efx;  in ef4_init_rx_queue()
739 	netif_dbg(rx_queue->efx, drv, rx_queue->efx->net_dev,  in ef4_init_rx_queue()
740 		  "initialising RX queue %d\n", ef4_rx_queue_index(rx_queue));  in ef4_init_rx_queue()
743 	rx_queue->added_count = 0;  in ef4_init_rx_queue()
744 	rx_queue->notified_count = 0;  in ef4_init_rx_queue()
745 	rx_queue->removed_count = 0;  in ef4_init_rx_queue()
746 	rx_queue->min_fill = -1U;  in ef4_init_rx_queue()
747 	ef4_init_rx_recycle_ring(efx, rx_queue);  in ef4_init_rx_queue()
749 	rx_queue->page_remove = 0;  in ef4_init_rx_queue()
750 	rx_queue->page_add = rx_queue->page_ptr_mask + 1;  in ef4_init_rx_queue()
751 	rx_queue->page_recycle_count = 0;  in ef4_init_rx_queue()
752 	rx_queue->page_recycle_failed = 0;  in ef4_init_rx_queue()
753 	rx_queue->page_recycle_full = 0;  in ef4_init_rx_queue()
767 	rx_queue->max_fill = max_fill;  in ef4_init_rx_queue()
768 	rx_queue->fast_fill_trigger = trigger;  in ef4_init_rx_queue()
769 	rx_queue->refill_enabled = true;  in ef4_init_rx_queue()
772 	ef4_nic_init_rx(rx_queue);  in ef4_init_rx_queue()
775 void ef4_fini_rx_queue(struct ef4_rx_queue *rx_queue)  in ef4_fini_rx_queue()  argument
778 	struct ef4_nic *efx = rx_queue->efx;  in ef4_fini_rx_queue()
781 	netif_dbg(rx_queue->efx, drv, rx_queue->efx->net_dev,  in ef4_fini_rx_queue()
782 		  "shutting down RX queue %d\n", ef4_rx_queue_index(rx_queue));  in ef4_fini_rx_queue()
784 	del_timer_sync(&rx_queue->slow_fill);  in ef4_fini_rx_queue()
787 	if (rx_queue->buffer) {  in ef4_fini_rx_queue()
788 		for (i = rx_queue->removed_count; i < rx_queue->added_count;  in ef4_fini_rx_queue()
790 			unsigned index = i & rx_queue->ptr_mask;  in ef4_fini_rx_queue()
791 			rx_buf = ef4_rx_buffer(rx_queue, index);  in ef4_fini_rx_queue()
792 			ef4_fini_rx_buffer(rx_queue, rx_buf);  in ef4_fini_rx_queue()
797 	for (i = 0; i <= rx_queue->page_ptr_mask; i++) {  in ef4_fini_rx_queue()
798 		struct page *page = rx_queue->page_ring[i];  in ef4_fini_rx_queue()
810 	kfree(rx_queue->page_ring);  in ef4_fini_rx_queue()
811 	rx_queue->page_ring = NULL;  in ef4_fini_rx_queue()
814 void ef4_remove_rx_queue(struct ef4_rx_queue *rx_queue)  in ef4_remove_rx_queue()  argument
816 	netif_dbg(rx_queue->efx, drv, rx_queue->efx->net_dev,  in ef4_remove_rx_queue()
817 		  "destroying RX queue %d\n", ef4_rx_queue_index(rx_queue));  in ef4_remove_rx_queue()
819 	ef4_nic_remove_rx(rx_queue);  in ef4_remove_rx_queue()
821 	kfree(rx_queue->buffer);  in ef4_remove_rx_queue()
822 	rx_queue->buffer = NULL;  in ef4_remove_rx_queue()