65 static dma_addr_t nfp_net_dma_map_rx(struct nfp_net_dp *dp, void *frag)  in nfp_net_dma_map_rx()  argument
67 	return dma_map_single_attrs(dp->dev, frag + NFP_NET_RX_BUF_HEADROOM,  in nfp_net_dma_map_rx()
978 	const skb_frag_t *frag;  in nfp_net_tx()  local
1063 			frag = &skb_shinfo(skb)->frags[f];  in nfp_net_tx()
1064 			fsize = skb_frag_size(frag);  in nfp_net_tx()
1066 			dma_addr = skb_frag_dma_map(dp->dev, frag, 0,  in nfp_net_tx()
1105 		frag = &skb_shinfo(skb)->frags[f];  in nfp_net_tx()
1107 			       skb_frag_size(frag), DMA_TO_DEVICE);  in nfp_net_tx()
1158 		const skb_frag_t *frag;  in nfp_net_tx_complete()  local
1183 			frag = &skb_shinfo(skb)->frags[fidx];  in nfp_net_tx_complete()
1185 				       skb_frag_size(frag), DMA_TO_DEVICE);  in nfp_net_tx_complete()
1273 	const skb_frag_t *frag;  in nfp_net_tx_ring_reset()  local
1293 			frag = &skb_shinfo(skb)->frags[tx_buf->fidx];  in nfp_net_tx_ring_reset()
1295 				       skb_frag_size(frag), DMA_TO_DEVICE);  in nfp_net_tx_ring_reset()
1358 nfp_net_free_frag(void *frag, bool xdp)  in nfp_net_free_frag()  argument
1361 		skb_free_frag(frag);  in nfp_net_free_frag()
1363 		__free_page(virt_to_page(frag));  in nfp_net_free_frag()
1377 	void *frag;  in nfp_net_rx_alloc_one()  local
1380 		frag = netdev_alloc_frag(dp->fl_bufsz);  in nfp_net_rx_alloc_one()
1385 		frag = page ? page_address(page) : NULL;  in nfp_net_rx_alloc_one()
1387 	if (!frag) {  in nfp_net_rx_alloc_one()
1392 	*dma_addr = nfp_net_dma_map_rx(dp, frag);  in nfp_net_rx_alloc_one()
1394 		nfp_net_free_frag(frag, dp->xdp_prog);  in nfp_net_rx_alloc_one()
1399 	return frag;  in nfp_net_rx_alloc_one()
1404 	void *frag;  in nfp_net_napi_alloc_one()  local
1407 		frag = napi_alloc_frag(dp->fl_bufsz);  in nfp_net_napi_alloc_one()
1408 		if (unlikely(!frag))  in nfp_net_napi_alloc_one()
1416 		frag = page_address(page);  in nfp_net_napi_alloc_one()
1419 	*dma_addr = nfp_net_dma_map_rx(dp, frag);  in nfp_net_napi_alloc_one()
1421 		nfp_net_free_frag(frag, dp->xdp_prog);  in nfp_net_napi_alloc_one()
1426 	return frag;  in nfp_net_napi_alloc_one()
1438 				void *frag, dma_addr_t dma_addr)  in nfp_net_rx_give_one()  argument
1447 	rx_ring->rxbufs[wr_idx].frag = frag;  in nfp_net_rx_give_one()
1486 	rx_ring->rxbufs[wr_idx].frag = rx_ring->rxbufs[last_idx].frag;  in nfp_net_rx_ring_reset()
1488 	rx_ring->rxbufs[last_idx].frag = NULL;  in nfp_net_rx_ring_reset()
1515 		if (!rx_ring->rxbufs[i].frag)  in nfp_net_rx_ring_bufs_free()
1519 		nfp_net_free_frag(rx_ring->rxbufs[i].frag, dp->xdp_prog);  in nfp_net_rx_ring_bufs_free()
1521 		rx_ring->rxbufs[i].frag = NULL;  in nfp_net_rx_ring_bufs_free()
1540 		rxbufs[i].frag = nfp_net_rx_alloc_one(dp, &rxbufs[i].dma_addr);  in nfp_net_rx_ring_bufs_alloc()
1541 		if (!rxbufs[i].frag) {  in nfp_net_rx_ring_bufs_alloc()
1562 		nfp_net_rx_give_one(dp, rx_ring, rx_ring->rxbufs[i].frag,  in nfp_net_rx_ring_fill_freelist()
1728 	if (skb && rxbuf && skb->head == rxbuf->frag)  in nfp_net_rx_drop()
1729 		page_ref_inc(virt_to_head_page(rxbuf->frag));  in nfp_net_rx_drop()
1731 		nfp_net_rx_give_one(dp, rx_ring, rxbuf->frag, rxbuf->dma_addr);  in nfp_net_rx_drop()
1764 	nfp_net_rx_give_one(dp, rx_ring, txbuf->frag, txbuf->dma_addr);  in nfp_net_tx_xdp_buf()
1766 	txbuf->frag = rxbuf->frag;  in nfp_net_tx_xdp_buf()
1891 					      rxbuf->frag + meta_off, rxd);  in nfp_net_rx()
1896 						 rxbuf->frag + meta_off,  in nfp_net_rx()
1898 			if (unlikely(end != rxbuf->frag + pkt_off)) {  in nfp_net_rx()
1907 			void *orig_data = rxbuf->frag + pkt_off;  in nfp_net_rx()
1911 			xdp.data_hard_start = rxbuf->frag + NFP_NET_RX_BUF_HEADROOM;  in nfp_net_rx()
1942 				nfp_net_rx_give_one(dp, rx_ring, rxbuf->frag,  in nfp_net_rx()
1953 			nfp_app_ctrl_rx_raw(nn->app, rxbuf->frag + pkt_off,  in nfp_net_rx()
1955 			nfp_net_rx_give_one(dp, rx_ring, rxbuf->frag,  in nfp_net_rx()
1974 		skb = build_skb(rxbuf->frag, true_bufsz);  in nfp_net_rx()
2241 	if (unlikely(!nfp_ctrl_meta_ok(nn, rxbuf->frag + meta_off, meta_len))) {  in nfp_ctrl_rx_one()
2248 	skb = build_skb(rxbuf->frag, dp->fl_bufsz);  in nfp_ctrl_rx_one()
2415 		if (!tx_ring->txbufs[i].frag)  in nfp_net_tx_ring_bufs_free()
2419 		__free_page(virt_to_page(tx_ring->txbufs[i].frag));  in nfp_net_tx_ring_bufs_free()
2434 		txbufs[i].frag = nfp_net_rx_alloc_one(dp, &txbufs[i].dma_addr);  in nfp_net_tx_ring_bufs_alloc()
2435 		if (!txbufs[i].frag) {  in nfp_net_tx_ring_bufs_alloc()