39 nfp_nfdk_tx_tso(struct nfp_net_r_vector *r_vec, struct nfp_nfdk_tx_buf *txbuf,  in nfp_nfdk_tx_tso()  argument
69 	u64_stats_update_begin(&r_vec->tx_sync);  in nfp_nfdk_tx_tso()
70 	r_vec->tx_lso++;  in nfp_nfdk_tx_tso()
71 	u64_stats_update_end(&r_vec->tx_sync);  in nfp_nfdk_tx_tso()
77 nfp_nfdk_tx_csum(struct nfp_net_dp *dp, struct nfp_net_r_vector *r_vec,  in nfp_nfdk_tx_csum()  argument
102 	u64_stats_update_begin(&r_vec->tx_sync);  in nfp_nfdk_tx_csum()
104 		r_vec->hw_csum_tx += pkt_cnt;  in nfp_nfdk_tx_csum()
107 		r_vec->hw_csum_tx_inner += pkt_cnt;  in nfp_nfdk_tx_csum()
109 	u64_stats_update_end(&r_vec->tx_sync);  in nfp_nfdk_tx_csum()
237 	struct nfp_net_r_vector *r_vec;  in nfp_nfdk_tx()  local
251 	r_vec = tx_ring->r_vec;  in nfp_nfdk_tx()
260 		u64_stats_update_begin(&r_vec->tx_sync);  in nfp_nfdk_tx()
261 		r_vec->tx_busy++;  in nfp_nfdk_tx()
262 		u64_stats_update_end(&r_vec->tx_sync);  in nfp_nfdk_tx()
367 		metadata = nfp_nfdk_tx_csum(dp, r_vec, 1, skb, metadata);  in nfp_nfdk_tx()
372 		(txd + 1)->raw = nfp_nfdk_tx_tso(r_vec, txbuf, skb);  in nfp_nfdk_tx()
375 		metadata = nfp_nfdk_tx_csum(dp, r_vec, txbuf->pkt_cnt, skb, metadata);  in nfp_nfdk_tx()
431 	u64_stats_update_begin(&r_vec->tx_sync);  in nfp_nfdk_tx()
432 	r_vec->tx_errors++;  in nfp_nfdk_tx()
433 	u64_stats_update_end(&r_vec->tx_sync);  in nfp_nfdk_tx()
445 	struct nfp_net_r_vector *r_vec = tx_ring->r_vec;  in nfp_nfdk_tx_complete()  local
446 	struct nfp_net_dp *dp = &r_vec->nfp_net->dp;  in nfp_nfdk_tx_complete()
519 	u64_stats_update_begin(&r_vec->tx_sync);  in nfp_nfdk_tx_complete()
520 	r_vec->tx_bytes += done_bytes;  in nfp_nfdk_tx_complete()
521 	r_vec->tx_pkts += done_pkts;  in nfp_nfdk_tx_complete()
522 	u64_stats_update_end(&r_vec->tx_sync);  in nfp_nfdk_tx_complete()
647 nfp_nfdk_rx_csum(struct nfp_net_dp *dp, struct nfp_net_r_vector *r_vec,  in nfp_nfdk_rx_csum()  argument
659 		u64_stats_update_begin(&r_vec->rx_sync);  in nfp_nfdk_rx_csum()
660 		r_vec->hw_csum_rx_complete++;  in nfp_nfdk_rx_csum()
661 		u64_stats_update_end(&r_vec->rx_sync);  in nfp_nfdk_rx_csum()
666 		u64_stats_update_begin(&r_vec->rx_sync);  in nfp_nfdk_rx_csum()
667 		r_vec->hw_csum_rx_error++;  in nfp_nfdk_rx_csum()
668 		u64_stats_update_end(&r_vec->rx_sync);  in nfp_nfdk_rx_csum()
679 		u64_stats_update_begin(&r_vec->rx_sync);  in nfp_nfdk_rx_csum()
680 		r_vec->hw_csum_rx_ok++;  in nfp_nfdk_rx_csum()
681 		u64_stats_update_end(&r_vec->rx_sync);  in nfp_nfdk_rx_csum()
687 		u64_stats_update_begin(&r_vec->rx_sync);  in nfp_nfdk_rx_csum()
688 		r_vec->hw_csum_rx_inner_ok++;  in nfp_nfdk_rx_csum()
689 		u64_stats_update_end(&r_vec->rx_sync);  in nfp_nfdk_rx_csum()
774 nfp_nfdk_rx_drop(const struct nfp_net_dp *dp, struct nfp_net_r_vector *r_vec,  in nfp_nfdk_rx_drop()  argument
778 	u64_stats_update_begin(&r_vec->rx_sync);  in nfp_nfdk_rx_drop()
779 	r_vec->rx_drops++;  in nfp_nfdk_rx_drop()
784 		r_vec->rx_replace_buf_alloc_fail++;  in nfp_nfdk_rx_drop()
785 	u64_stats_update_end(&r_vec->rx_sync);  in nfp_nfdk_rx_drop()
800 	struct nfp_net_r_vector *r_vec = tx_ring->r_vec;  in nfp_nfdk_xdp_complete()  local
801 	struct nfp_net_dp *dp = &r_vec->nfp_net->dp;  in nfp_nfdk_xdp_complete()
817 	rx_ring = r_vec->rx_ring;  in nfp_nfdk_xdp_complete()
842 		u64_stats_update_begin(&r_vec->tx_sync);  in nfp_nfdk_xdp_complete()
844 		r_vec->tx_pkts++;  in nfp_nfdk_xdp_complete()
845 		u64_stats_update_end(&r_vec->tx_sync);  in nfp_nfdk_xdp_complete()
890 			nfp_nfdk_rx_drop(dp, rx_ring->r_vec, rx_ring, rxbuf,  in nfp_nfdk_tx_xdp_buf()
992 	struct nfp_net_r_vector *r_vec = rx_ring->r_vec;  in nfp_nfdk_rx()  local
993 	struct nfp_net_dp *dp = &r_vec->nfp_net->dp;  in nfp_nfdk_rx()
1007 	tx_ring = r_vec->xdp_ring;  in nfp_nfdk_rx()
1061 		u64_stats_update_begin(&r_vec->rx_sync);  in nfp_nfdk_rx()
1062 		r_vec->rx_pkts++;  in nfp_nfdk_rx()
1063 		r_vec->rx_bytes += pkt_len;  in nfp_nfdk_rx()
1064 		u64_stats_update_end(&r_vec->rx_sync);  in nfp_nfdk_rx()
1070 			nfp_nfdk_rx_drop(dp, r_vec, rx_ring, rxbuf, NULL);  in nfp_nfdk_rx()
1083 				nfp_nfdk_rx_drop(dp, r_vec, rx_ring, rxbuf,  in nfp_nfdk_rx()
1149 				nfp_nfdk_rx_drop(dp, r_vec, rx_ring, rxbuf,  in nfp_nfdk_rx()
1160 			nfp_nfdk_rx_drop(dp, r_vec, rx_ring, rxbuf, NULL);  in nfp_nfdk_rx()
1165 			nfp_nfdk_rx_drop(dp, r_vec, rx_ring, rxbuf, skb);  in nfp_nfdk_rx()
1182 		nfp_nfdk_rx_csum(dp, r_vec, rxd, &meta, skb);  in nfp_nfdk_rx()
1185 			nfp_nfdk_rx_drop(dp, r_vec, rx_ring, NULL, skb);  in nfp_nfdk_rx()
1193 			napi_gro_receive(&rx_ring->r_vec->napi, skb);  in nfp_nfdk_rx()
1223 	struct nfp_net_r_vector *r_vec =  in nfp_nfdk_poll()  local
1227 	if (r_vec->tx_ring)  in nfp_nfdk_poll()
1228 		nfp_nfdk_tx_complete(r_vec->tx_ring, budget);  in nfp_nfdk_poll()
1229 	if (r_vec->rx_ring)  in nfp_nfdk_poll()
1230 		pkts_polled = nfp_nfdk_rx(r_vec->rx_ring, budget);  in nfp_nfdk_poll()
1234 			nfp_net_irq_unmask(r_vec->nfp_net, r_vec->irq_entry);  in nfp_nfdk_poll()
1236 	if (r_vec->nfp_net->rx_coalesce_adapt_on && r_vec->rx_ring) {  in nfp_nfdk_poll()
1242 			start = u64_stats_fetch_begin(&r_vec->rx_sync);  in nfp_nfdk_poll()
1243 			pkts = r_vec->rx_pkts;  in nfp_nfdk_poll()
1244 			bytes = r_vec->rx_bytes;  in nfp_nfdk_poll()
1245 		} while (u64_stats_fetch_retry(&r_vec->rx_sync, start));  in nfp_nfdk_poll()
1247 		dim_update_sample(r_vec->event_ctr, pkts, bytes, &dim_sample);  in nfp_nfdk_poll()
1248 		net_dim(&r_vec->rx_dim, dim_sample);  in nfp_nfdk_poll()
1251 	if (r_vec->nfp_net->tx_coalesce_adapt_on && r_vec->tx_ring) {  in nfp_nfdk_poll()
1257 			start = u64_stats_fetch_begin(&r_vec->tx_sync);  in nfp_nfdk_poll()
1258 			pkts = r_vec->tx_pkts;  in nfp_nfdk_poll()
1259 			bytes = r_vec->tx_bytes;  in nfp_nfdk_poll()
1260 		} while (u64_stats_fetch_retry(&r_vec->tx_sync, start));  in nfp_nfdk_poll()
1262 		dim_update_sample(r_vec->event_ctr, pkts, bytes, &dim_sample);  in nfp_nfdk_poll()
1263 		net_dim(&r_vec->tx_dim, dim_sample);  in nfp_nfdk_poll()
1273 nfp_nfdk_ctrl_tx_one(struct nfp_net *nn, struct nfp_net_r_vector *r_vec,  in nfp_nfdk_ctrl_tx_one()  argument
1286 	dp = &r_vec->nfp_net->dp;  in nfp_nfdk_ctrl_tx_one()
1287 	tx_ring = r_vec->tx_ring;  in nfp_nfdk_ctrl_tx_one()
1296 		u64_stats_update_begin(&r_vec->tx_sync);  in nfp_nfdk_ctrl_tx_one()
1297 		r_vec->tx_busy++;  in nfp_nfdk_ctrl_tx_one()
1298 		u64_stats_update_end(&r_vec->tx_sync);  in nfp_nfdk_ctrl_tx_one()
1300 			__skb_queue_tail(&r_vec->queue, skb);  in nfp_nfdk_ctrl_tx_one()
1302 			__skb_queue_head(&r_vec->queue, skb);  in nfp_nfdk_ctrl_tx_one()
1401 	u64_stats_update_begin(&r_vec->tx_sync);  in nfp_nfdk_ctrl_tx_one()
1402 	r_vec->tx_errors++;  in nfp_nfdk_ctrl_tx_one()
1403 	u64_stats_update_end(&r_vec->tx_sync);  in nfp_nfdk_ctrl_tx_one()
1408 static void __nfp_ctrl_tx_queued(struct nfp_net_r_vector *r_vec)  in __nfp_ctrl_tx_queued()  argument
1412 	while ((skb = __skb_dequeue(&r_vec->queue)))  in __nfp_ctrl_tx_queued()
1413 		if (nfp_nfdk_ctrl_tx_one(r_vec->nfp_net, r_vec, skb, true))  in __nfp_ctrl_tx_queued()
1437 		struct nfp_net_r_vector *r_vec, struct nfp_net_rx_ring *rx_ring)  in nfp_ctrl_rx_one()  argument
1473 	u64_stats_update_begin(&r_vec->rx_sync);  in nfp_ctrl_rx_one()
1474 	r_vec->rx_pkts++;  in nfp_ctrl_rx_one()
1475 	r_vec->rx_bytes += pkt_len;  in nfp_ctrl_rx_one()
1476 	u64_stats_update_end(&r_vec->rx_sync);  in nfp_ctrl_rx_one()
1483 		nfp_nfdk_rx_drop(dp, r_vec, rx_ring, rxbuf, NULL);  in nfp_ctrl_rx_one()
1489 		nfp_nfdk_rx_drop(dp, r_vec, rx_ring, rxbuf, NULL);  in nfp_ctrl_rx_one()
1494 		nfp_nfdk_rx_drop(dp, r_vec, rx_ring, rxbuf, skb);  in nfp_ctrl_rx_one()
1510 static bool nfp_ctrl_rx(struct nfp_net_r_vector *r_vec)  in nfp_ctrl_rx()  argument
1512 	struct nfp_net_rx_ring *rx_ring = r_vec->rx_ring;  in nfp_ctrl_rx()
1513 	struct nfp_net *nn = r_vec->nfp_net;  in nfp_ctrl_rx()
1517 	while (nfp_ctrl_rx_one(nn, dp, r_vec, rx_ring) && budget--)  in nfp_ctrl_rx()
1525 	struct nfp_net_r_vector *r_vec = from_tasklet(r_vec, t, tasklet);  in nfp_nfdk_ctrl_poll()  local
1527 	spin_lock(&r_vec->lock);  in nfp_nfdk_ctrl_poll()
1528 	nfp_nfdk_tx_complete(r_vec->tx_ring, 0);  in nfp_nfdk_ctrl_poll()
1529 	__nfp_ctrl_tx_queued(r_vec);  in nfp_nfdk_ctrl_poll()
1530 	spin_unlock(&r_vec->lock);  in nfp_nfdk_ctrl_poll()
1532 	if (nfp_ctrl_rx(r_vec)) {  in nfp_nfdk_ctrl_poll()
1533 		nfp_net_irq_unmask(r_vec->nfp_net, r_vec->irq_entry);  in nfp_nfdk_ctrl_poll()
1535 		tasklet_schedule(&r_vec->tasklet);  in nfp_nfdk_ctrl_poll()
1536 		nn_dp_warn(&r_vec->nfp_net->dp,  in nfp_nfdk_ctrl_poll()