53 static struct sk_buff *enetc_tx_swbd_get_skb(struct enetc_tx_swbd *tx_swbd)  in enetc_tx_swbd_get_skb()  argument
55 	if (tx_swbd->is_xdp_tx || tx_swbd->is_xdp_redirect)  in enetc_tx_swbd_get_skb()
58 	return tx_swbd->skb;  in enetc_tx_swbd_get_skb()
62 enetc_tx_swbd_get_xdp_frame(struct enetc_tx_swbd *tx_swbd)  in enetc_tx_swbd_get_xdp_frame()  argument
64 	if (tx_swbd->is_xdp_redirect)  in enetc_tx_swbd_get_xdp_frame()
65 		return tx_swbd->xdp_frame;  in enetc_tx_swbd_get_xdp_frame()
71 				struct enetc_tx_swbd *tx_swbd)  in enetc_unmap_tx_buff()  argument
77 	if (tx_swbd->is_dma_page)  in enetc_unmap_tx_buff()
78 		dma_unmap_page(tx_ring->dev, tx_swbd->dma,  in enetc_unmap_tx_buff()
79 			       tx_swbd->is_xdp_tx ? PAGE_SIZE : tx_swbd->len,  in enetc_unmap_tx_buff()
80 			       tx_swbd->dir);  in enetc_unmap_tx_buff()
82 		dma_unmap_single(tx_ring->dev, tx_swbd->dma,  in enetc_unmap_tx_buff()
83 				 tx_swbd->len, tx_swbd->dir);  in enetc_unmap_tx_buff()
84 	tx_swbd->dma = 0;  in enetc_unmap_tx_buff()
88 				struct enetc_tx_swbd *tx_swbd)  in enetc_free_tx_frame()  argument
90 	struct xdp_frame *xdp_frame = enetc_tx_swbd_get_xdp_frame(tx_swbd);  in enetc_free_tx_frame()
91 	struct sk_buff *skb = enetc_tx_swbd_get_skb(tx_swbd);  in enetc_free_tx_frame()
93 	if (tx_swbd->dma)  in enetc_free_tx_frame()
94 		enetc_unmap_tx_buff(tx_ring, tx_swbd);  in enetc_free_tx_frame()
97 		xdp_return_frame(tx_swbd->xdp_frame);  in enetc_free_tx_frame()
98 		tx_swbd->xdp_frame = NULL;  in enetc_free_tx_frame()
101 		tx_swbd->skb = NULL;  in enetc_free_tx_frame()
150 	struct enetc_tx_swbd *tx_swbd;  in enetc_map_tx_buffs()  local
174 	tx_swbd = &tx_ring->tx_swbd[i];  in enetc_map_tx_buffs()
175 	tx_swbd->dma = dma;  in enetc_map_tx_buffs()
176 	tx_swbd->len = len;  in enetc_map_tx_buffs()
177 	tx_swbd->is_dma_page = 0;  in enetc_map_tx_buffs()
178 	tx_swbd->dir = DMA_TO_DEVICE;  in enetc_map_tx_buffs()
193 	tx_swbd->do_twostep_tstamp = do_twostep_tstamp;  in enetc_map_tx_buffs()
194 	tx_swbd->qbv_en = !!(priv->active_offloads & ENETC_F_QBV);  in enetc_map_tx_buffs()
195 	tx_swbd->check_wb = tx_swbd->do_twostep_tstamp || tx_swbd->qbv_en;  in enetc_map_tx_buffs()
218 		tx_swbd++;  in enetc_map_tx_buffs()
223 			tx_swbd = tx_ring->tx_swbd;  in enetc_map_tx_buffs()
288 		tx_swbd++;  in enetc_map_tx_buffs()
293 			tx_swbd = tx_ring->tx_swbd;  in enetc_map_tx_buffs()
301 		tx_swbd->dma = dma;  in enetc_map_tx_buffs()
302 		tx_swbd->len = len;  in enetc_map_tx_buffs()
303 		tx_swbd->is_dma_page = 1;  in enetc_map_tx_buffs()
304 		tx_swbd->dir = DMA_TO_DEVICE;  in enetc_map_tx_buffs()
313 	tx_ring->tx_swbd[i].is_eof = true;  in enetc_map_tx_buffs()
314 	tx_ring->tx_swbd[i].skb = skb;  in enetc_map_tx_buffs()
329 		tx_swbd = &tx_ring->tx_swbd[i];  in enetc_map_tx_buffs()
330 		enetc_free_tx_frame(tx_ring, tx_swbd);  in enetc_map_tx_buffs()
340 				 struct enetc_tx_swbd *tx_swbd,  in enetc_map_tx_tso_hdr()  argument
365 	tx_swbd->len = hdr_len;  in enetc_map_tx_tso_hdr()
366 	tx_swbd->do_twostep_tstamp = false;  in enetc_map_tx_tso_hdr()
367 	tx_swbd->check_wb = false;  in enetc_map_tx_tso_hdr()
377 		tx_swbd = &tx_ring->tx_swbd[*i];  in enetc_map_tx_tso_hdr()
393 				 struct enetc_tx_swbd *tx_swbd,  in enetc_map_tx_tso_data()  argument
411 		tx_swbd->is_eof = 1;  in enetc_map_tx_tso_data()
418 	tx_swbd->dma = addr;  in enetc_map_tx_tso_data()
419 	tx_swbd->len = size;  in enetc_map_tx_tso_data()
420 	tx_swbd->dir = DMA_TO_DEVICE;  in enetc_map_tx_tso_data()
491 	struct enetc_tx_swbd *tx_swbd;  in enetc_map_tx_tso_buffs()  local
508 		tx_swbd = &tx_ring->tx_swbd[i];  in enetc_map_tx_tso_buffs()
521 		enetc_map_tx_tso_hdr(tx_ring, skb, tx_swbd, txbd, &i, hdr_len, data_len);  in enetc_map_tx_tso_buffs()
533 			tx_swbd = &tx_ring->tx_swbd[i];  in enetc_map_tx_tso_buffs()
544 			err = enetc_map_tx_tso_data(tx_ring, skb, tx_swbd, txbd,  in enetc_map_tx_tso_buffs()
562 			tx_swbd->skb = skb;  in enetc_map_tx_tso_buffs()
578 		tx_swbd = &tx_ring->tx_swbd[i];  in enetc_map_tx_tso_buffs()
579 		enetc_free_tx_frame(tx_ring, tx_swbd);  in enetc_map_tx_tso_buffs()
776 				      struct enetc_tx_swbd *tx_swbd)  in enetc_recycle_xdp_tx_buff()  argument
780 		.dma = tx_swbd->dma,  in enetc_recycle_xdp_tx_buff()
781 		.page = tx_swbd->page,  in enetc_recycle_xdp_tx_buff()
782 		.page_offset = tx_swbd->page_offset,  in enetc_recycle_xdp_tx_buff()
783 		.dir = tx_swbd->dir,  in enetc_recycle_xdp_tx_buff()
784 		.len = tx_swbd->len,  in enetc_recycle_xdp_tx_buff()
819 	struct enetc_tx_swbd *tx_swbd;  in enetc_clean_tx_ring()  local
825 	tx_swbd = &tx_ring->tx_swbd[i];  in enetc_clean_tx_ring()
832 		struct xdp_frame *xdp_frame = enetc_tx_swbd_get_xdp_frame(tx_swbd);  in enetc_clean_tx_ring()
833 		struct sk_buff *skb = enetc_tx_swbd_get_skb(tx_swbd);  in enetc_clean_tx_ring()
834 		bool is_eof = tx_swbd->is_eof;  in enetc_clean_tx_ring()
836 		if (unlikely(tx_swbd->check_wb)) {  in enetc_clean_tx_ring()
840 			    tx_swbd->do_twostep_tstamp) {  in enetc_clean_tx_ring()
846 			if (tx_swbd->qbv_en &&  in enetc_clean_tx_ring()
851 		if (tx_swbd->is_xdp_tx)  in enetc_clean_tx_ring()
852 			enetc_recycle_xdp_tx_buff(tx_ring, tx_swbd);  in enetc_clean_tx_ring()
853 		else if (likely(tx_swbd->dma))  in enetc_clean_tx_ring()
854 			enetc_unmap_tx_buff(tx_ring, tx_swbd);  in enetc_clean_tx_ring()
872 		tx_byte_cnt += tx_swbd->len;  in enetc_clean_tx_ring()
876 		memset(tx_swbd, 0, sizeof(*tx_swbd));  in enetc_clean_tx_ring()
879 		tx_swbd++;  in enetc_clean_tx_ring()
883 			tx_swbd = tx_ring->tx_swbd;  in enetc_clean_tx_ring()
1253 				  struct enetc_tx_swbd *tx_swbd,  in enetc_xdp_map_tx_buff()  argument
1261 	txbd->addr = cpu_to_le64(tx_swbd->dma + tx_swbd->page_offset);  in enetc_xdp_map_tx_buff()
1262 	txbd->buf_len = cpu_to_le16(tx_swbd->len);  in enetc_xdp_map_tx_buff()
1265 	memcpy(&tx_ring->tx_swbd[i], tx_swbd, sizeof(*tx_swbd));  in enetc_xdp_map_tx_buff()
1500 		struct enetc_tx_swbd *tx_swbd = &xdp_tx_arr[n];  in enetc_rx_swbd_to_xdp_tx_swbd()  local
1503 		tx_swbd->dma = rx_swbd->dma;  in enetc_rx_swbd_to_xdp_tx_swbd()
1504 		tx_swbd->dir = rx_swbd->dir;  in enetc_rx_swbd_to_xdp_tx_swbd()
1505 		tx_swbd->page = rx_swbd->page;  in enetc_rx_swbd_to_xdp_tx_swbd()
1506 		tx_swbd->page_offset = rx_swbd->page_offset;  in enetc_rx_swbd_to_xdp_tx_swbd()
1507 		tx_swbd->len = rx_swbd->len;  in enetc_rx_swbd_to_xdp_tx_swbd()
1508 		tx_swbd->is_dma_page = true;  in enetc_rx_swbd_to_xdp_tx_swbd()
1509 		tx_swbd->is_xdp_tx = true;  in enetc_rx_swbd_to_xdp_tx_swbd()
1510 		tx_swbd->is_eof = false;  in enetc_rx_swbd_to_xdp_tx_swbd()
1792 	res->tx_swbd = vcalloc(bd_count, sizeof(*res->tx_swbd));  in enetc_alloc_tx_resource()
1793 	if (!res->tx_swbd)  in enetc_alloc_tx_resource()
1813 	vfree(res->tx_swbd);  in enetc_alloc_tx_resource()
1814 	res->tx_swbd = NULL;  in enetc_alloc_tx_resource()
1824 	vfree(res->tx_swbd);  in enetc_free_tx_resource()
1945 	tx_ring->tx_swbd = res ? res->tx_swbd : NULL;  in enetc_assign_tx_resource()
1995 		struct enetc_tx_swbd *tx_swbd = &tx_ring->tx_swbd[i];  in enetc_free_tx_ring()  local
1997 		enetc_free_tx_frame(tx_ring, tx_swbd);  in enetc_free_tx_ring()