73 nfp_nfd3_tx_tso(struct nfp_net_r_vector *r_vec, struct nfp_nfd3_tx_buf *txbuf,  in nfp_nfd3_tx_tso()  argument
102 	u64_stats_update_begin(&r_vec->tx_sync);  in nfp_nfd3_tx_tso()
103 	r_vec->tx_lso++;  in nfp_nfd3_tx_tso()
104 	u64_stats_update_end(&r_vec->tx_sync);  in nfp_nfd3_tx_tso()
119 nfp_nfd3_tx_csum(struct nfp_net_dp *dp, struct nfp_net_r_vector *r_vec,  in nfp_nfd3_tx_csum()  argument
162 	u64_stats_update_begin(&r_vec->tx_sync);  in nfp_nfd3_tx_csum()
164 		r_vec->hw_csum_tx_inner += txbuf->pkt_cnt;  in nfp_nfd3_tx_csum()
166 		r_vec->hw_csum_tx += txbuf->pkt_cnt;  in nfp_nfd3_tx_csum()
167 	u64_stats_update_end(&r_vec->tx_sync);  in nfp_nfd3_tx_csum()
240 	struct nfp_net_r_vector *r_vec;  in nfp_nfd3_tx()  local
254 	r_vec = tx_ring->r_vec;  in nfp_nfd3_tx()
264 		u64_stats_update_begin(&r_vec->tx_sync);  in nfp_nfd3_tx()
265 		r_vec->tx_busy++;  in nfp_nfd3_tx()
266 		u64_stats_update_end(&r_vec->tx_sync);  in nfp_nfd3_tx()
270 	skb = nfp_net_tls_tx(dp, r_vec, skb, &tls_handle, &nr_frags);  in nfp_nfd3_tx()
308 	nfp_nfd3_tx_tso(r_vec, txbuf, txd, skb, md_bytes);  in nfp_nfd3_tx()
309 	nfp_nfd3_tx_csum(dp, r_vec, txbuf, txd, skb);  in nfp_nfd3_tx()
344 		u64_stats_update_begin(&r_vec->tx_sync);  in nfp_nfd3_tx()
345 		r_vec->tx_gather++;  in nfp_nfd3_tx()
346 		u64_stats_update_end(&r_vec->tx_sync);  in nfp_nfd3_tx()
384 	u64_stats_update_begin(&r_vec->tx_sync);  in nfp_nfd3_tx()
385 	r_vec->tx_errors++;  in nfp_nfd3_tx()
386 	u64_stats_update_end(&r_vec->tx_sync);  in nfp_nfd3_tx()
399 	struct nfp_net_r_vector *r_vec = tx_ring->r_vec;  in nfp_nfd3_tx_complete()  local
400 	struct nfp_net_dp *dp = &r_vec->nfp_net->dp;  in nfp_nfd3_tx_complete()
459 	u64_stats_update_begin(&r_vec->tx_sync);  in nfp_nfd3_tx_complete()
460 	r_vec->tx_bytes += done_bytes;  in nfp_nfd3_tx_complete()
461 	r_vec->tx_pkts += done_pkts;  in nfp_nfd3_tx_complete()
462 	u64_stats_update_end(&r_vec->tx_sync);  in nfp_nfd3_tx_complete()
484 	struct nfp_net_r_vector *r_vec = tx_ring->r_vec;  in nfp_nfd3_xdp_complete()  local
485 	struct nfp_net_dp *dp = &r_vec->nfp_net->dp;  in nfp_nfd3_xdp_complete()
512 	u64_stats_update_begin(&r_vec->tx_sync);  in nfp_nfd3_xdp_complete()
513 	r_vec->tx_bytes += done_bytes;  in nfp_nfd3_xdp_complete()
514 	r_vec->tx_pkts += done_pkts;  in nfp_nfd3_xdp_complete()
515 	u64_stats_update_end(&r_vec->tx_sync);  in nfp_nfd3_xdp_complete()
638 nfp_nfd3_rx_csum(const struct nfp_net_dp *dp, struct nfp_net_r_vector *r_vec,  in nfp_nfd3_rx_csum()  argument
650 		u64_stats_update_begin(&r_vec->rx_sync);  in nfp_nfd3_rx_csum()
651 		r_vec->hw_csum_rx_complete++;  in nfp_nfd3_rx_csum()
652 		u64_stats_update_end(&r_vec->rx_sync);  in nfp_nfd3_rx_csum()
657 		u64_stats_update_begin(&r_vec->rx_sync);  in nfp_nfd3_rx_csum()
658 		r_vec->hw_csum_rx_error++;  in nfp_nfd3_rx_csum()
659 		u64_stats_update_end(&r_vec->rx_sync);  in nfp_nfd3_rx_csum()
670 		u64_stats_update_begin(&r_vec->rx_sync);  in nfp_nfd3_rx_csum()
671 		r_vec->hw_csum_rx_ok++;  in nfp_nfd3_rx_csum()
672 		u64_stats_update_end(&r_vec->rx_sync);  in nfp_nfd3_rx_csum()
678 		u64_stats_update_begin(&r_vec->rx_sync);  in nfp_nfd3_rx_csum()
679 		r_vec->hw_csum_rx_inner_ok++;  in nfp_nfd3_rx_csum()
680 		u64_stats_update_end(&r_vec->rx_sync);  in nfp_nfd3_rx_csum()
778 nfp_nfd3_rx_drop(const struct nfp_net_dp *dp, struct nfp_net_r_vector *r_vec,  in nfp_nfd3_rx_drop()  argument
782 	u64_stats_update_begin(&r_vec->rx_sync);  in nfp_nfd3_rx_drop()
783 	r_vec->rx_drops++;  in nfp_nfd3_rx_drop()
788 		r_vec->rx_replace_buf_alloc_fail++;  in nfp_nfd3_rx_drop()
789 	u64_stats_update_end(&r_vec->rx_sync);  in nfp_nfd3_rx_drop()
824 			nfp_nfd3_rx_drop(dp, rx_ring->r_vec, rx_ring, rxbuf,  in nfp_nfd3_tx_xdp_buf()
875 	struct nfp_net_r_vector *r_vec = rx_ring->r_vec;  in nfp_nfd3_rx()  local
876 	struct nfp_net_dp *dp = &r_vec->nfp_net->dp;  in nfp_nfd3_rx()
890 	tx_ring = r_vec->xdp_ring;  in nfp_nfd3_rx()
944 		u64_stats_update_begin(&r_vec->rx_sync);  in nfp_nfd3_rx()
945 		r_vec->rx_pkts++;  in nfp_nfd3_rx()
946 		r_vec->rx_bytes += pkt_len;  in nfp_nfd3_rx()
947 		u64_stats_update_end(&r_vec->rx_sync);  in nfp_nfd3_rx()
953 			nfp_nfd3_rx_drop(dp, r_vec, rx_ring, rxbuf, NULL);  in nfp_nfd3_rx()
969 				nfp_nfd3_rx_drop(dp, r_vec, rx_ring, rxbuf,  in nfp_nfd3_rx()
1035 				nfp_nfd3_rx_drop(dp, r_vec, rx_ring, rxbuf,  in nfp_nfd3_rx()
1046 			nfp_nfd3_rx_drop(dp, r_vec, rx_ring, rxbuf, NULL);  in nfp_nfd3_rx()
1051 			nfp_nfd3_rx_drop(dp, r_vec, rx_ring, rxbuf, skb);  in nfp_nfd3_rx()
1068 		nfp_nfd3_rx_csum(dp, r_vec, rxd, &meta, skb);  in nfp_nfd3_rx()
1073 			u64_stats_update_begin(&r_vec->rx_sync);  in nfp_nfd3_rx()
1074 			r_vec->hw_tls_rx++;  in nfp_nfd3_rx()
1075 			u64_stats_update_end(&r_vec->rx_sync);  in nfp_nfd3_rx()
1080 			nfp_nfd3_rx_drop(dp, r_vec, rx_ring, NULL, skb);  in nfp_nfd3_rx()
1088 			napi_gro_receive(&rx_ring->r_vec->napi, skb);  in nfp_nfd3_rx()
1118 	struct nfp_net_r_vector *r_vec =  in nfp_nfd3_poll()  local
1122 	if (r_vec->tx_ring)  in nfp_nfd3_poll()
1123 		nfp_nfd3_tx_complete(r_vec->tx_ring, budget);  in nfp_nfd3_poll()
1124 	if (r_vec->rx_ring)  in nfp_nfd3_poll()
1125 		pkts_polled = nfp_nfd3_rx(r_vec->rx_ring, budget);  in nfp_nfd3_poll()
1129 			nfp_net_irq_unmask(r_vec->nfp_net, r_vec->irq_entry);  in nfp_nfd3_poll()
1131 	if (r_vec->nfp_net->rx_coalesce_adapt_on && r_vec->rx_ring) {  in nfp_nfd3_poll()
1137 			start = u64_stats_fetch_begin(&r_vec->rx_sync);  in nfp_nfd3_poll()
1138 			pkts = r_vec->rx_pkts;  in nfp_nfd3_poll()
1139 			bytes = r_vec->rx_bytes;  in nfp_nfd3_poll()
1140 		} while (u64_stats_fetch_retry(&r_vec->rx_sync, start));  in nfp_nfd3_poll()
1142 		dim_update_sample(r_vec->event_ctr, pkts, bytes, &dim_sample);  in nfp_nfd3_poll()
1143 		net_dim(&r_vec->rx_dim, dim_sample);  in nfp_nfd3_poll()
1146 	if (r_vec->nfp_net->tx_coalesce_adapt_on && r_vec->tx_ring) {  in nfp_nfd3_poll()
1152 			start = u64_stats_fetch_begin(&r_vec->tx_sync);  in nfp_nfd3_poll()
1153 			pkts = r_vec->tx_pkts;  in nfp_nfd3_poll()
1154 			bytes = r_vec->tx_bytes;  in nfp_nfd3_poll()
1155 		} while (u64_stats_fetch_retry(&r_vec->tx_sync, start));  in nfp_nfd3_poll()
1157 		dim_update_sample(r_vec->event_ctr, pkts, bytes, &dim_sample);  in nfp_nfd3_poll()
1158 		net_dim(&r_vec->tx_dim, dim_sample);  in nfp_nfd3_poll()
1168 nfp_nfd3_ctrl_tx_one(struct nfp_net *nn, struct nfp_net_r_vector *r_vec,  in nfp_nfd3_ctrl_tx_one()  argument
1179 	dp = &r_vec->nfp_net->dp;  in nfp_nfd3_ctrl_tx_one()
1180 	tx_ring = r_vec->tx_ring;  in nfp_nfd3_ctrl_tx_one()
1188 		u64_stats_update_begin(&r_vec->tx_sync);  in nfp_nfd3_ctrl_tx_one()
1189 		r_vec->tx_busy++;  in nfp_nfd3_ctrl_tx_one()
1190 		u64_stats_update_end(&r_vec->tx_sync);  in nfp_nfd3_ctrl_tx_one()
1192 			__skb_queue_tail(&r_vec->queue, skb);  in nfp_nfd3_ctrl_tx_one()
1194 			__skb_queue_head(&r_vec->queue, skb);  in nfp_nfd3_ctrl_tx_one()
1244 	u64_stats_update_begin(&r_vec->tx_sync);  in nfp_nfd3_ctrl_tx_one()
1245 	r_vec->tx_errors++;  in nfp_nfd3_ctrl_tx_one()
1246 	u64_stats_update_end(&r_vec->tx_sync);  in nfp_nfd3_ctrl_tx_one()
1251 static void __nfp_ctrl_tx_queued(struct nfp_net_r_vector *r_vec)  in __nfp_ctrl_tx_queued()  argument
1255 	while ((skb = __skb_dequeue(&r_vec->queue)))  in __nfp_ctrl_tx_queued()
1256 		if (nfp_nfd3_ctrl_tx_one(r_vec->nfp_net, r_vec, skb, true))  in __nfp_ctrl_tx_queued()
1280 		struct nfp_net_r_vector *r_vec, struct nfp_net_rx_ring *rx_ring)  in nfp_ctrl_rx_one()  argument
1316 	u64_stats_update_begin(&r_vec->rx_sync);  in nfp_ctrl_rx_one()
1317 	r_vec->rx_pkts++;  in nfp_ctrl_rx_one()
1318 	r_vec->rx_bytes += pkt_len;  in nfp_ctrl_rx_one()
1319 	u64_stats_update_end(&r_vec->rx_sync);  in nfp_ctrl_rx_one()
1326 		nfp_nfd3_rx_drop(dp, r_vec, rx_ring, rxbuf, NULL);  in nfp_ctrl_rx_one()
1332 		nfp_nfd3_rx_drop(dp, r_vec, rx_ring, rxbuf, NULL);  in nfp_ctrl_rx_one()
1337 		nfp_nfd3_rx_drop(dp, r_vec, rx_ring, rxbuf, skb);  in nfp_ctrl_rx_one()
1353 static bool nfp_ctrl_rx(struct nfp_net_r_vector *r_vec)  in nfp_ctrl_rx()  argument
1355 	struct nfp_net_rx_ring *rx_ring = r_vec->rx_ring;  in nfp_ctrl_rx()
1356 	struct nfp_net *nn = r_vec->nfp_net;  in nfp_ctrl_rx()
1360 	while (nfp_ctrl_rx_one(nn, dp, r_vec, rx_ring) && budget--)  in nfp_ctrl_rx()
1368 	struct nfp_net_r_vector *r_vec = from_tasklet(r_vec, t, tasklet);  in nfp_nfd3_ctrl_poll()  local
1370 	spin_lock(&r_vec->lock);  in nfp_nfd3_ctrl_poll()
1371 	nfp_nfd3_tx_complete(r_vec->tx_ring, 0);  in nfp_nfd3_ctrl_poll()
1372 	__nfp_ctrl_tx_queued(r_vec);  in nfp_nfd3_ctrl_poll()
1373 	spin_unlock(&r_vec->lock);  in nfp_nfd3_ctrl_poll()
1375 	if (nfp_ctrl_rx(r_vec)) {  in nfp_nfd3_ctrl_poll()
1376 		nfp_net_irq_unmask(r_vec->nfp_net, r_vec->irq_entry);  in nfp_nfd3_ctrl_poll()
1378 		tasklet_schedule(&r_vec->tasklet);  in nfp_nfd3_ctrl_poll()
1379 		nn_dp_warn(&r_vec->nfp_net->dp,  in nfp_nfd3_ctrl_poll()