34 static struct sk_buff *enetc_tx_swbd_get_skb(struct enetc_tx_swbd *tx_swbd)  in enetc_tx_swbd_get_skb()  argument
36 	if (tx_swbd->is_xdp_tx || tx_swbd->is_xdp_redirect)  in enetc_tx_swbd_get_skb()
39 	return tx_swbd->skb;  in enetc_tx_swbd_get_skb()
43 enetc_tx_swbd_get_xdp_frame(struct enetc_tx_swbd *tx_swbd)  in enetc_tx_swbd_get_xdp_frame()  argument
45 	if (tx_swbd->is_xdp_redirect)  in enetc_tx_swbd_get_xdp_frame()
46 		return tx_swbd->xdp_frame;  in enetc_tx_swbd_get_xdp_frame()
52 				struct enetc_tx_swbd *tx_swbd)  in enetc_unmap_tx_buff()  argument
58 	if (tx_swbd->is_dma_page)  in enetc_unmap_tx_buff()
59 		dma_unmap_page(tx_ring->dev, tx_swbd->dma,  in enetc_unmap_tx_buff()
60 			       tx_swbd->is_xdp_tx ? PAGE_SIZE : tx_swbd->len,  in enetc_unmap_tx_buff()
61 			       tx_swbd->dir);  in enetc_unmap_tx_buff()
63 		dma_unmap_single(tx_ring->dev, tx_swbd->dma,  in enetc_unmap_tx_buff()
64 				 tx_swbd->len, tx_swbd->dir);  in enetc_unmap_tx_buff()
65 	tx_swbd->dma = 0;  in enetc_unmap_tx_buff()
69 				struct enetc_tx_swbd *tx_swbd)  in enetc_free_tx_frame()  argument
71 	struct xdp_frame *xdp_frame = enetc_tx_swbd_get_xdp_frame(tx_swbd);  in enetc_free_tx_frame()
72 	struct sk_buff *skb = enetc_tx_swbd_get_skb(tx_swbd);  in enetc_free_tx_frame()
74 	if (tx_swbd->dma)  in enetc_free_tx_frame()
75 		enetc_unmap_tx_buff(tx_ring, tx_swbd);  in enetc_free_tx_frame()
78 		xdp_return_frame(tx_swbd->xdp_frame);  in enetc_free_tx_frame()
79 		tx_swbd->xdp_frame = NULL;  in enetc_free_tx_frame()
82 		tx_swbd->skb = NULL;  in enetc_free_tx_frame()
131 	struct enetc_tx_swbd *tx_swbd;  in enetc_map_tx_buffs()  local
155 	tx_swbd = &tx_ring->tx_swbd[i];  in enetc_map_tx_buffs()
156 	tx_swbd->dma = dma;  in enetc_map_tx_buffs()
157 	tx_swbd->len = len;  in enetc_map_tx_buffs()
158 	tx_swbd->is_dma_page = 0;  in enetc_map_tx_buffs()
159 	tx_swbd->dir = DMA_TO_DEVICE;  in enetc_map_tx_buffs()
174 	tx_swbd->do_twostep_tstamp = do_twostep_tstamp;  in enetc_map_tx_buffs()
175 	tx_swbd->qbv_en = !!(priv->active_offloads & ENETC_F_QBV);  in enetc_map_tx_buffs()
176 	tx_swbd->check_wb = tx_swbd->do_twostep_tstamp || tx_swbd->qbv_en;  in enetc_map_tx_buffs()
199 		tx_swbd++;  in enetc_map_tx_buffs()
204 			tx_swbd = tx_ring->tx_swbd;  in enetc_map_tx_buffs()
269 		tx_swbd++;  in enetc_map_tx_buffs()
274 			tx_swbd = tx_ring->tx_swbd;  in enetc_map_tx_buffs()
282 		tx_swbd->dma = dma;  in enetc_map_tx_buffs()
283 		tx_swbd->len = len;  in enetc_map_tx_buffs()
284 		tx_swbd->is_dma_page = 1;  in enetc_map_tx_buffs()
285 		tx_swbd->dir = DMA_TO_DEVICE;  in enetc_map_tx_buffs()
294 	tx_ring->tx_swbd[i].is_eof = true;  in enetc_map_tx_buffs()
295 	tx_ring->tx_swbd[i].skb = skb;  in enetc_map_tx_buffs()
310 		tx_swbd = &tx_ring->tx_swbd[i];  in enetc_map_tx_buffs()
311 		enetc_free_tx_frame(tx_ring, tx_swbd);  in enetc_map_tx_buffs()
321 				 struct enetc_tx_swbd *tx_swbd,  in enetc_map_tx_tso_hdr()  argument
346 	tx_swbd->len = hdr_len;  in enetc_map_tx_tso_hdr()
347 	tx_swbd->do_twostep_tstamp = false;  in enetc_map_tx_tso_hdr()
348 	tx_swbd->check_wb = false;  in enetc_map_tx_tso_hdr()
358 		tx_swbd = &tx_ring->tx_swbd[*i];  in enetc_map_tx_tso_hdr()
374 				 struct enetc_tx_swbd *tx_swbd,  in enetc_map_tx_tso_data()  argument
392 		tx_swbd->is_eof = 1;  in enetc_map_tx_tso_data()
399 	tx_swbd->dma = addr;  in enetc_map_tx_tso_data()
400 	tx_swbd->len = size;  in enetc_map_tx_tso_data()
401 	tx_swbd->dir = DMA_TO_DEVICE;  in enetc_map_tx_tso_data()
472 	struct enetc_tx_swbd *tx_swbd;  in enetc_map_tx_tso_buffs()  local
489 		tx_swbd = &tx_ring->tx_swbd[i];  in enetc_map_tx_tso_buffs()
502 		enetc_map_tx_tso_hdr(tx_ring, skb, tx_swbd, txbd, &i, hdr_len, data_len);  in enetc_map_tx_tso_buffs()
514 			tx_swbd = &tx_ring->tx_swbd[i];  in enetc_map_tx_tso_buffs()
525 			err = enetc_map_tx_tso_data(tx_ring, skb, tx_swbd, txbd,  in enetc_map_tx_tso_buffs()
543 			tx_swbd->skb = skb;  in enetc_map_tx_tso_buffs()
559 		tx_swbd = &tx_ring->tx_swbd[i];  in enetc_map_tx_tso_buffs()
560 		enetc_free_tx_frame(tx_ring, tx_swbd);  in enetc_map_tx_tso_buffs()
756 				      struct enetc_tx_swbd *tx_swbd)  in enetc_recycle_xdp_tx_buff()  argument
760 		.dma = tx_swbd->dma,  in enetc_recycle_xdp_tx_buff()
761 		.page = tx_swbd->page,  in enetc_recycle_xdp_tx_buff()
762 		.page_offset = tx_swbd->page_offset,  in enetc_recycle_xdp_tx_buff()
763 		.dir = tx_swbd->dir,  in enetc_recycle_xdp_tx_buff()
764 		.len = tx_swbd->len,  in enetc_recycle_xdp_tx_buff()
799 	struct enetc_tx_swbd *tx_swbd;  in enetc_clean_tx_ring()  local
805 	tx_swbd = &tx_ring->tx_swbd[i];  in enetc_clean_tx_ring()
812 		struct xdp_frame *xdp_frame = enetc_tx_swbd_get_xdp_frame(tx_swbd);  in enetc_clean_tx_ring()
813 		struct sk_buff *skb = enetc_tx_swbd_get_skb(tx_swbd);  in enetc_clean_tx_ring()
814 		bool is_eof = tx_swbd->is_eof;  in enetc_clean_tx_ring()
816 		if (unlikely(tx_swbd->check_wb)) {  in enetc_clean_tx_ring()
820 			    tx_swbd->do_twostep_tstamp) {  in enetc_clean_tx_ring()
826 			if (tx_swbd->qbv_en &&  in enetc_clean_tx_ring()
831 		if (tx_swbd->is_xdp_tx)  in enetc_clean_tx_ring()
832 			enetc_recycle_xdp_tx_buff(tx_ring, tx_swbd);  in enetc_clean_tx_ring()
833 		else if (likely(tx_swbd->dma))  in enetc_clean_tx_ring()
834 			enetc_unmap_tx_buff(tx_ring, tx_swbd);  in enetc_clean_tx_ring()
852 		tx_byte_cnt += tx_swbd->len;  in enetc_clean_tx_ring()
856 		memset(tx_swbd, 0, sizeof(*tx_swbd));  in enetc_clean_tx_ring()
859 		tx_swbd++;  in enetc_clean_tx_ring()
863 			tx_swbd = tx_ring->tx_swbd;  in enetc_clean_tx_ring()
1227 				  struct enetc_tx_swbd *tx_swbd,  in enetc_xdp_map_tx_buff()  argument
1235 	txbd->addr = cpu_to_le64(tx_swbd->dma + tx_swbd->page_offset);  in enetc_xdp_map_tx_buff()
1236 	txbd->buf_len = cpu_to_le16(tx_swbd->len);  in enetc_xdp_map_tx_buff()
1239 	memcpy(&tx_ring->tx_swbd[i], tx_swbd, sizeof(*tx_swbd));  in enetc_xdp_map_tx_buff()
1462 		struct enetc_tx_swbd *tx_swbd = &xdp_tx_arr[n];  in enetc_rx_swbd_to_xdp_tx_swbd()  local
1465 		tx_swbd->dma = rx_swbd->dma;  in enetc_rx_swbd_to_xdp_tx_swbd()
1466 		tx_swbd->dir = rx_swbd->dir;  in enetc_rx_swbd_to_xdp_tx_swbd()
1467 		tx_swbd->page = rx_swbd->page;  in enetc_rx_swbd_to_xdp_tx_swbd()
1468 		tx_swbd->page_offset = rx_swbd->page_offset;  in enetc_rx_swbd_to_xdp_tx_swbd()
1469 		tx_swbd->len = rx_swbd->len;  in enetc_rx_swbd_to_xdp_tx_swbd()
1470 		tx_swbd->is_dma_page = true;  in enetc_rx_swbd_to_xdp_tx_swbd()
1471 		tx_swbd->is_xdp_tx = true;  in enetc_rx_swbd_to_xdp_tx_swbd()
1472 		tx_swbd->is_eof = false;  in enetc_rx_swbd_to_xdp_tx_swbd()
1760 	txr->tx_swbd = vzalloc(txr->bd_count * sizeof(struct enetc_tx_swbd));  in enetc_alloc_txbdr()
1761 	if (!txr->tx_swbd)  in enetc_alloc_txbdr()
1787 	vfree(txr->tx_swbd);  in enetc_alloc_txbdr()
1788 	txr->tx_swbd = NULL;  in enetc_alloc_txbdr()
1798 		enetc_free_tx_frame(txr, &txr->tx_swbd[i]);  in enetc_free_txbdr()
1809 	vfree(txr->tx_swbd);  in enetc_free_txbdr()
1810 	txr->tx_swbd = NULL;  in enetc_free_txbdr()
1913 	if (!tx_ring->tx_swbd)  in enetc_free_tx_ring()
1917 		struct enetc_tx_swbd *tx_swbd = &tx_ring->tx_swbd[i];  in enetc_free_tx_ring()  local
1919 		enetc_free_tx_frame(tx_ring, tx_swbd);  in enetc_free_tx_ring()