474 ef4_farch_build_rx_desc(struct ef4_rx_queue *rx_queue, unsigned index)  in ef4_farch_build_rx_desc()  argument
479 	rxd = ef4_rx_desc(rx_queue, index);  in ef4_farch_build_rx_desc()
480 	rx_buf = ef4_rx_buffer(rx_queue, index);  in ef4_farch_build_rx_desc()
484 			     rx_queue->efx->type->rx_buffer_padding,  in ef4_farch_build_rx_desc()
492 void ef4_farch_rx_write(struct ef4_rx_queue *rx_queue)  in ef4_farch_rx_write()  argument
494 	struct ef4_nic *efx = rx_queue->efx;  in ef4_farch_rx_write()
498 	while (rx_queue->notified_count != rx_queue->added_count) {  in ef4_farch_rx_write()
500 			rx_queue,  in ef4_farch_rx_write()
501 			rx_queue->notified_count & rx_queue->ptr_mask);  in ef4_farch_rx_write()
502 		++rx_queue->notified_count;  in ef4_farch_rx_write()
506 	write_ptr = rx_queue->added_count & rx_queue->ptr_mask;  in ef4_farch_rx_write()
509 			ef4_rx_queue_index(rx_queue));  in ef4_farch_rx_write()
512 int ef4_farch_rx_probe(struct ef4_rx_queue *rx_queue)  in ef4_farch_rx_probe()  argument
514 	struct ef4_nic *efx = rx_queue->efx;  in ef4_farch_rx_probe()
517 	entries = rx_queue->ptr_mask + 1;  in ef4_farch_rx_probe()
518 	return ef4_alloc_special_buffer(efx, &rx_queue->rxd,  in ef4_farch_rx_probe()
522 void ef4_farch_rx_init(struct ef4_rx_queue *rx_queue)  in ef4_farch_rx_init()  argument
525 	struct ef4_nic *efx = rx_queue->efx;  in ef4_farch_rx_init()
539 		  ef4_rx_queue_index(rx_queue), rx_queue->rxd.index,  in ef4_farch_rx_init()
540 		  rx_queue->rxd.index + rx_queue->rxd.entries - 1);  in ef4_farch_rx_init()
542 	rx_queue->scatter_n = 0;  in ef4_farch_rx_init()
545 	ef4_init_special_buffer(efx, &rx_queue->rxd);  in ef4_farch_rx_init()
551 			      FRF_AZ_RX_DESCQ_BUF_BASE_ID, rx_queue->rxd.index,  in ef4_farch_rx_init()
553 			      ef4_rx_queue_channel(rx_queue)->channel,  in ef4_farch_rx_init()
556 			      ef4_rx_queue_index(rx_queue),  in ef4_farch_rx_init()
558 			      __ffs(rx_queue->rxd.entries),  in ef4_farch_rx_init()
563 			 ef4_rx_queue_index(rx_queue));  in ef4_farch_rx_init()
566 static void ef4_farch_flush_rx_queue(struct ef4_rx_queue *rx_queue)  in ef4_farch_flush_rx_queue()  argument
568 	struct ef4_nic *efx = rx_queue->efx;  in ef4_farch_flush_rx_queue()
574 			     ef4_rx_queue_index(rx_queue));  in ef4_farch_flush_rx_queue()
578 void ef4_farch_rx_fini(struct ef4_rx_queue *rx_queue)  in ef4_farch_rx_fini()  argument
581 	struct ef4_nic *efx = rx_queue->efx;  in ef4_farch_rx_fini()
586 			 ef4_rx_queue_index(rx_queue));  in ef4_farch_rx_fini()
589 	ef4_fini_special_buffer(efx, &rx_queue->rxd);  in ef4_farch_rx_fini()
593 void ef4_farch_rx_remove(struct ef4_rx_queue *rx_queue)  in ef4_farch_rx_remove()  argument
595 	ef4_free_special_buffer(rx_queue->efx, &rx_queue->rxd);  in ef4_farch_rx_remove()
665 	struct ef4_rx_queue *rx_queue;  in ef4_farch_do_flush()  local
673 		ef4_for_each_channel_rx_queue(rx_queue, channel) {  in ef4_farch_do_flush()
674 			rx_queue->flush_pending = true;  in ef4_farch_do_flush()
685 			ef4_for_each_channel_rx_queue(rx_queue, channel) {  in ef4_farch_do_flush()
690 				if (rx_queue->flush_pending) {  in ef4_farch_do_flush()
691 					rx_queue->flush_pending = false;  in ef4_farch_do_flush()
694 					ef4_farch_flush_rx_queue(rx_queue);  in ef4_farch_do_flush()
724 	struct ef4_rx_queue *rx_queue;  in ef4_farch_fini_dmaq()  local
737 			ef4_for_each_channel_rx_queue(rx_queue, channel)  in ef4_farch_fini_dmaq()
738 				ef4_farch_rx_fini(rx_queue);  in ef4_farch_fini_dmaq()
871 static u16 ef4_farch_handle_rx_not_ok(struct ef4_rx_queue *rx_queue,  in ef4_farch_handle_rx_not_ok()  argument
874 	struct ef4_channel *channel = ef4_rx_queue_channel(rx_queue);  in ef4_farch_handle_rx_not_ok()
875 	struct ef4_nic *efx = rx_queue->efx;  in ef4_farch_handle_rx_not_ok()
926 			  ef4_rx_queue_index(rx_queue), EF4_QWORD_VAL(*event),  in ef4_farch_handle_rx_not_ok()
951 ef4_farch_handle_rx_bad_index(struct ef4_rx_queue *rx_queue, unsigned index)  in ef4_farch_handle_rx_bad_index()  argument
953 	struct ef4_channel *channel = ef4_rx_queue_channel(rx_queue);  in ef4_farch_handle_rx_bad_index()
954 	struct ef4_nic *efx = rx_queue->efx;  in ef4_farch_handle_rx_bad_index()
957 	if (rx_queue->scatter_n &&  in ef4_farch_handle_rx_bad_index()
958 	    index == ((rx_queue->removed_count + rx_queue->scatter_n - 1) &  in ef4_farch_handle_rx_bad_index()
959 		      rx_queue->ptr_mask)) {  in ef4_farch_handle_rx_bad_index()
964 	expected = rx_queue->removed_count & rx_queue->ptr_mask;  in ef4_farch_handle_rx_bad_index()
965 	dropped = (index - expected) & rx_queue->ptr_mask;  in ef4_farch_handle_rx_bad_index()
990 	struct ef4_rx_queue *rx_queue;  in ef4_farch_handle_rx_event()  local
1001 	rx_queue = ef4_channel_get_rx_queue(channel);  in ef4_farch_handle_rx_event()
1004 	expected_ptr = ((rx_queue->removed_count + rx_queue->scatter_n) &  in ef4_farch_handle_rx_event()
1005 			rx_queue->ptr_mask);  in ef4_farch_handle_rx_event()
1009 	    unlikely(rx_ev_sop != (rx_queue->scatter_n == 0))) {  in ef4_farch_handle_rx_event()
1011 		    !ef4_farch_handle_rx_bad_index(rx_queue, rx_ev_desc_ptr))  in ef4_farch_handle_rx_event()
1015 		if (rx_queue->scatter_n) {  in ef4_farch_handle_rx_event()
1017 				rx_queue,  in ef4_farch_handle_rx_event()
1018 				rx_queue->removed_count & rx_queue->ptr_mask,  in ef4_farch_handle_rx_event()
1019 				rx_queue->scatter_n, 0, EF4_RX_PKT_DISCARD);  in ef4_farch_handle_rx_event()
1020 			rx_queue->removed_count += rx_queue->scatter_n;  in ef4_farch_handle_rx_event()
1021 			rx_queue->scatter_n = 0;  in ef4_farch_handle_rx_event()
1031 				rx_queue,  in ef4_farch_handle_rx_event()
1032 				rx_queue->removed_count & rx_queue->ptr_mask,  in ef4_farch_handle_rx_event()
1034 			++rx_queue->removed_count;  in ef4_farch_handle_rx_event()
1039 	++rx_queue->scatter_n;  in ef4_farch_handle_rx_event()
1064 		flags = ef4_farch_handle_rx_not_ok(rx_queue, event);  in ef4_farch_handle_rx_event()
1082 	ef4_rx_packet(rx_queue,  in ef4_farch_handle_rx_event()
1083 		      rx_queue->removed_count & rx_queue->ptr_mask,  in ef4_farch_handle_rx_event()
1084 		      rx_queue->scatter_n, rx_ev_byte_cnt, flags);  in ef4_farch_handle_rx_event()
1085 	rx_queue->removed_count += rx_queue->scatter_n;  in ef4_farch_handle_rx_event()
1086 	rx_queue->scatter_n = 0;  in ef4_farch_handle_rx_event()
1118 	struct ef4_rx_queue *rx_queue;  in ef4_farch_handle_rx_flush_done()  local
1129 	rx_queue = ef4_channel_get_rx_queue(channel);  in ef4_farch_handle_rx_flush_done()
1134 		rx_queue->flush_pending = true;  in ef4_farch_handle_rx_flush_done()
1137 		ef4_farch_magic_event(ef4_rx_queue_channel(rx_queue),  in ef4_farch_handle_rx_flush_done()
1138 				      EF4_CHANNEL_MAGIC_RX_DRAIN(rx_queue));  in ef4_farch_handle_rx_flush_done()
1160 	struct ef4_rx_queue *rx_queue =  in ef4_farch_handle_generated_event()  local
1170 	} else if (rx_queue && magic == EF4_CHANNEL_MAGIC_FILL(rx_queue)) {  in ef4_farch_handle_generated_event()
1174 		ef4_fast_push_rx_descriptors(rx_queue, true);  in ef4_farch_handle_generated_event()
1175 	} else if (rx_queue && magic == EF4_CHANNEL_MAGIC_RX_DRAIN(rx_queue)) {  in ef4_farch_handle_generated_event()
1394 void ef4_farch_rx_defer_refill(struct ef4_rx_queue *rx_queue)  in ef4_farch_rx_defer_refill()  argument
1396 	ef4_farch_magic_event(ef4_rx_queue_channel(rx_queue),  in ef4_farch_rx_defer_refill()
1397 			      EF4_CHANNEL_MAGIC_FILL(rx_queue));  in ef4_farch_rx_defer_refill()