66 static dma_addr_t nfp_net_dma_map_rx(struct nfp_net_dp *dp, void *frag)  in nfp_net_dma_map_rx()  argument
68 	return dma_map_single_attrs(dp->dev, frag + NFP_NET_RX_BUF_HEADROOM,  in nfp_net_dma_map_rx()
980 	const skb_frag_t *frag;  in nfp_net_tx()  local
1065 			frag = &skb_shinfo(skb)->frags[f];  in nfp_net_tx()
1066 			fsize = skb_frag_size(frag);  in nfp_net_tx()
1068 			dma_addr = skb_frag_dma_map(dp->dev, frag, 0,  in nfp_net_tx()
1107 		frag = &skb_shinfo(skb)->frags[f];  in nfp_net_tx()
1109 			       skb_frag_size(frag), DMA_TO_DEVICE);  in nfp_net_tx()
1160 		const skb_frag_t *frag;  in nfp_net_tx_complete()  local
1185 			frag = &skb_shinfo(skb)->frags[fidx];  in nfp_net_tx_complete()
1187 				       skb_frag_size(frag), DMA_TO_DEVICE);  in nfp_net_tx_complete()
1275 	const skb_frag_t *frag;  in nfp_net_tx_ring_reset()  local
1295 			frag = &skb_shinfo(skb)->frags[tx_buf->fidx];  in nfp_net_tx_ring_reset()
1297 				       skb_frag_size(frag), DMA_TO_DEVICE);  in nfp_net_tx_ring_reset()
1354 nfp_net_free_frag(void *frag, bool xdp)  in nfp_net_free_frag()  argument
1357 		skb_free_frag(frag);  in nfp_net_free_frag()
1359 		__free_page(virt_to_page(frag));  in nfp_net_free_frag()
1373 	void *frag;  in nfp_net_rx_alloc_one()  local
1376 		frag = netdev_alloc_frag(dp->fl_bufsz);  in nfp_net_rx_alloc_one()
1381 		frag = page ? page_address(page) : NULL;  in nfp_net_rx_alloc_one()
1383 	if (!frag) {  in nfp_net_rx_alloc_one()
1388 	*dma_addr = nfp_net_dma_map_rx(dp, frag);  in nfp_net_rx_alloc_one()
1390 		nfp_net_free_frag(frag, dp->xdp_prog);  in nfp_net_rx_alloc_one()
1395 	return frag;  in nfp_net_rx_alloc_one()
1400 	void *frag;  in nfp_net_napi_alloc_one()  local
1403 		frag = napi_alloc_frag(dp->fl_bufsz);  in nfp_net_napi_alloc_one()
1404 		if (unlikely(!frag))  in nfp_net_napi_alloc_one()
1412 		frag = page_address(page);  in nfp_net_napi_alloc_one()
1415 	*dma_addr = nfp_net_dma_map_rx(dp, frag);  in nfp_net_napi_alloc_one()
1417 		nfp_net_free_frag(frag, dp->xdp_prog);  in nfp_net_napi_alloc_one()
1422 	return frag;  in nfp_net_napi_alloc_one()
1434 				void *frag, dma_addr_t dma_addr)  in nfp_net_rx_give_one()  argument
1443 	rx_ring->rxbufs[wr_idx].frag = frag;  in nfp_net_rx_give_one()
1482 	rx_ring->rxbufs[wr_idx].frag = rx_ring->rxbufs[last_idx].frag;  in nfp_net_rx_ring_reset()
1484 	rx_ring->rxbufs[last_idx].frag = NULL;  in nfp_net_rx_ring_reset()
1511 		if (!rx_ring->rxbufs[i].frag)  in nfp_net_rx_ring_bufs_free()
1515 		nfp_net_free_frag(rx_ring->rxbufs[i].frag, dp->xdp_prog);  in nfp_net_rx_ring_bufs_free()
1517 		rx_ring->rxbufs[i].frag = NULL;  in nfp_net_rx_ring_bufs_free()
1536 		rxbufs[i].frag = nfp_net_rx_alloc_one(dp, &rxbufs[i].dma_addr);  in nfp_net_rx_ring_bufs_alloc()
1537 		if (!rxbufs[i].frag) {  in nfp_net_rx_ring_bufs_alloc()
1558 		nfp_net_rx_give_one(dp, rx_ring, rx_ring->rxbufs[i].frag,  in nfp_net_rx_ring_fill_freelist()
1730 	if (skb && rxbuf && skb->head == rxbuf->frag)  in nfp_net_rx_drop()
1731 		page_ref_inc(virt_to_head_page(rxbuf->frag));  in nfp_net_rx_drop()
1733 		nfp_net_rx_give_one(dp, rx_ring, rxbuf->frag, rxbuf->dma_addr);  in nfp_net_rx_drop()
1771 	nfp_net_rx_give_one(dp, rx_ring, txbuf->frag, txbuf->dma_addr);  in nfp_net_tx_xdp_buf()
1773 	txbuf->frag = rxbuf->frag;  in nfp_net_tx_xdp_buf()
1899 					      rxbuf->frag + meta_off, rxd);  in nfp_net_rx()
1902 							rxbuf->frag + meta_off,  in nfp_net_rx()
1903 							rxbuf->frag + pkt_off,  in nfp_net_rx()
1913 			void *orig_data = rxbuf->frag + pkt_off;  in nfp_net_rx()
1917 			xdp.data_hard_start = rxbuf->frag + NFP_NET_RX_BUF_HEADROOM;  in nfp_net_rx()
1948 				nfp_net_rx_give_one(dp, rx_ring, rxbuf->frag,  in nfp_net_rx()
1959 			nfp_app_ctrl_rx_raw(nn->app, rxbuf->frag + pkt_off,  in nfp_net_rx()
1961 			nfp_net_rx_give_one(dp, rx_ring, rxbuf->frag,  in nfp_net_rx()
1980 		skb = build_skb(rxbuf->frag, true_bufsz);  in nfp_net_rx()
2247 	if (unlikely(!nfp_ctrl_meta_ok(nn, rxbuf->frag + meta_off, meta_len))) {  in nfp_ctrl_rx_one()
2254 	skb = build_skb(rxbuf->frag, dp->fl_bufsz);  in nfp_ctrl_rx_one()
2420 		if (!tx_ring->txbufs[i].frag)  in nfp_net_tx_ring_bufs_free()
2424 		__free_page(virt_to_page(tx_ring->txbufs[i].frag));  in nfp_net_tx_ring_bufs_free()
2439 		txbufs[i].frag = nfp_net_rx_alloc_one(dp, &txbufs[i].dma_addr);  in nfp_net_tx_ring_bufs_alloc()
2440 		if (!txbufs[i].frag) {  in nfp_net_tx_ring_bufs_alloc()