369 void ice_clean_rx_ring(struct ice_ring *rx_ring)  in ice_clean_rx_ring()  argument
371 	struct device *dev = rx_ring->dev;  in ice_clean_rx_ring()
375 	if (!rx_ring->rx_buf)  in ice_clean_rx_ring()
378 	if (rx_ring->xsk_pool) {  in ice_clean_rx_ring()
379 		ice_xsk_clean_rx_ring(rx_ring);  in ice_clean_rx_ring()
384 	for (i = 0; i < rx_ring->count; i++) {  in ice_clean_rx_ring()
385 		struct ice_rx_buf *rx_buf = &rx_ring->rx_buf[i];  in ice_clean_rx_ring()
399 					      rx_ring->rx_buf_len,  in ice_clean_rx_ring()
403 		dma_unmap_page_attrs(dev, rx_buf->dma, ice_rx_pg_size(rx_ring),  in ice_clean_rx_ring()
412 	memset(rx_ring->rx_buf, 0, sizeof(*rx_ring->rx_buf) * rx_ring->count);  in ice_clean_rx_ring()
415 	memset(rx_ring->desc, 0, rx_ring->size);  in ice_clean_rx_ring()
417 	rx_ring->next_to_alloc = 0;  in ice_clean_rx_ring()
418 	rx_ring->next_to_clean = 0;  in ice_clean_rx_ring()
419 	rx_ring->next_to_use = 0;  in ice_clean_rx_ring()
428 void ice_free_rx_ring(struct ice_ring *rx_ring)  in ice_free_rx_ring()  argument
430 	ice_clean_rx_ring(rx_ring);  in ice_free_rx_ring()
431 	if (rx_ring->vsi->type == ICE_VSI_PF)  in ice_free_rx_ring()
432 		if (xdp_rxq_info_is_reg(&rx_ring->xdp_rxq))  in ice_free_rx_ring()
433 			xdp_rxq_info_unreg(&rx_ring->xdp_rxq);  in ice_free_rx_ring()
434 	rx_ring->xdp_prog = NULL;  in ice_free_rx_ring()
435 	devm_kfree(rx_ring->dev, rx_ring->rx_buf);  in ice_free_rx_ring()
436 	rx_ring->rx_buf = NULL;  in ice_free_rx_ring()
438 	if (rx_ring->desc) {  in ice_free_rx_ring()
439 		dmam_free_coherent(rx_ring->dev, rx_ring->size,  in ice_free_rx_ring()
440 				   rx_ring->desc, rx_ring->dma);  in ice_free_rx_ring()
441 		rx_ring->desc = NULL;  in ice_free_rx_ring()
451 int ice_setup_rx_ring(struct ice_ring *rx_ring)  in ice_setup_rx_ring()  argument
453 	struct device *dev = rx_ring->dev;  in ice_setup_rx_ring()
459 	WARN_ON(rx_ring->rx_buf);  in ice_setup_rx_ring()
460 	rx_ring->rx_buf =  in ice_setup_rx_ring()
461 		devm_kzalloc(dev, sizeof(*rx_ring->rx_buf) * rx_ring->count,  in ice_setup_rx_ring()
463 	if (!rx_ring->rx_buf)  in ice_setup_rx_ring()
467 	rx_ring->size = ALIGN(rx_ring->count * sizeof(union ice_32byte_rx_desc),  in ice_setup_rx_ring()
469 	rx_ring->desc = dmam_alloc_coherent(dev, rx_ring->size, &rx_ring->dma,  in ice_setup_rx_ring()
471 	if (!rx_ring->desc) {  in ice_setup_rx_ring()
473 			rx_ring->size);  in ice_setup_rx_ring()
477 	rx_ring->next_to_use = 0;  in ice_setup_rx_ring()
478 	rx_ring->next_to_clean = 0;  in ice_setup_rx_ring()
480 	if (ice_is_xdp_ena_vsi(rx_ring->vsi))  in ice_setup_rx_ring()
481 		WRITE_ONCE(rx_ring->xdp_prog, rx_ring->vsi->xdp_prog);  in ice_setup_rx_ring()
483 	if (rx_ring->vsi->type == ICE_VSI_PF &&  in ice_setup_rx_ring()
484 	    !xdp_rxq_info_is_reg(&rx_ring->xdp_rxq))  in ice_setup_rx_ring()
485 		if (xdp_rxq_info_reg(&rx_ring->xdp_rxq, rx_ring->netdev,  in ice_setup_rx_ring()
486 				     rx_ring->q_index))  in ice_setup_rx_ring()
491 	devm_kfree(dev, rx_ring->rx_buf);  in ice_setup_rx_ring()
492 	rx_ring->rx_buf = NULL;  in ice_setup_rx_ring()
502 static unsigned int ice_rx_offset(struct ice_ring *rx_ring)  in ice_rx_offset()  argument
504 	if (ice_ring_uses_build_skb(rx_ring))  in ice_rx_offset()
506 	else if (ice_is_xdp_ena_vsi(rx_ring->vsi))  in ice_rx_offset()
513 ice_rx_frame_truesize(struct ice_ring *rx_ring, unsigned int __maybe_unused size)  in ice_rx_frame_truesize()  argument
518 	truesize = ice_rx_pg_size(rx_ring) / 2; /* Must be power-of-2 */  in ice_rx_frame_truesize()
520 	truesize = ice_rx_offset(rx_ring) ?  in ice_rx_frame_truesize()
521 		SKB_DATA_ALIGN(ice_rx_offset(rx_ring) + size) +  in ice_rx_frame_truesize()
537 ice_run_xdp(struct ice_ring *rx_ring, struct xdp_buff *xdp,  in ice_run_xdp()  argument
549 		xdp_ring = rx_ring->vsi->xdp_rings[smp_processor_id()];  in ice_run_xdp()
553 		err = xdp_do_redirect(rx_ring->netdev, xdp, xdp_prog);  in ice_run_xdp()
560 		trace_xdp_exception(rx_ring->netdev, xdp_prog, act);  in ice_run_xdp()
628 ice_alloc_mapped_page(struct ice_ring *rx_ring, struct ice_rx_buf *bi)  in ice_alloc_mapped_page()  argument
638 	page = dev_alloc_pages(ice_rx_pg_order(rx_ring));  in ice_alloc_mapped_page()
640 		rx_ring->rx_stats.alloc_page_failed++;  in ice_alloc_mapped_page()
645 	dma = dma_map_page_attrs(rx_ring->dev, page, 0, ice_rx_pg_size(rx_ring),  in ice_alloc_mapped_page()
651 	if (dma_mapping_error(rx_ring->dev, dma)) {  in ice_alloc_mapped_page()
652 		__free_pages(page, ice_rx_pg_order(rx_ring));  in ice_alloc_mapped_page()
653 		rx_ring->rx_stats.alloc_page_failed++;  in ice_alloc_mapped_page()
659 	bi->page_offset = ice_rx_offset(rx_ring);  in ice_alloc_mapped_page()
679 bool ice_alloc_rx_bufs(struct ice_ring *rx_ring, u16 cleaned_count)  in ice_alloc_rx_bufs()  argument
682 	u16 ntu = rx_ring->next_to_use;  in ice_alloc_rx_bufs()
686 	if ((!rx_ring->netdev && rx_ring->vsi->type != ICE_VSI_CTRL) ||  in ice_alloc_rx_bufs()
691 	rx_desc = ICE_RX_DESC(rx_ring, ntu);  in ice_alloc_rx_bufs()
692 	bi = &rx_ring->rx_buf[ntu];  in ice_alloc_rx_bufs()
696 		if (!ice_alloc_mapped_page(rx_ring, bi))  in ice_alloc_rx_bufs()
700 		dma_sync_single_range_for_device(rx_ring->dev, bi->dma,  in ice_alloc_rx_bufs()
702 						 rx_ring->rx_buf_len,  in ice_alloc_rx_bufs()
713 		if (unlikely(ntu == rx_ring->count)) {  in ice_alloc_rx_bufs()
714 			rx_desc = ICE_RX_DESC(rx_ring, 0);  in ice_alloc_rx_bufs()
715 			bi = rx_ring->rx_buf;  in ice_alloc_rx_bufs()
725 	if (rx_ring->next_to_use != ntu)  in ice_alloc_rx_bufs()
726 		ice_release_rx_desc(rx_ring, ntu);  in ice_alloc_rx_bufs()
817 ice_add_rx_frag(struct ice_ring *rx_ring, struct ice_rx_buf *rx_buf,  in ice_add_rx_frag()  argument
821 	unsigned int truesize = SKB_DATA_ALIGN(size + ice_rx_offset(rx_ring));  in ice_add_rx_frag()
823 	unsigned int truesize = ice_rx_pg_size(rx_ring) / 2;  in ice_add_rx_frag()
843 ice_reuse_rx_page(struct ice_ring *rx_ring, struct ice_rx_buf *old_buf)  in ice_reuse_rx_page()  argument
845 	u16 nta = rx_ring->next_to_alloc;  in ice_reuse_rx_page()
848 	new_buf = &rx_ring->rx_buf[nta];  in ice_reuse_rx_page()
852 	rx_ring->next_to_alloc = (nta < rx_ring->count) ? nta : 0;  in ice_reuse_rx_page()
875 ice_get_rx_buf(struct ice_ring *rx_ring, struct sk_buff **skb,  in ice_get_rx_buf()  argument
880 	rx_buf = &rx_ring->rx_buf[rx_ring->next_to_clean];  in ice_get_rx_buf()
893 	dma_sync_single_range_for_cpu(rx_ring->dev, rx_buf->dma,  in ice_get_rx_buf()
913 ice_build_skb(struct ice_ring *rx_ring, struct ice_rx_buf *rx_buf,  in ice_build_skb()  argument
918 	unsigned int truesize = ice_rx_pg_size(rx_ring) / 2;  in ice_build_skb()
940 	skb_record_rx_queue(skb, rx_ring->q_index);  in ice_build_skb()
965 ice_construct_skb(struct ice_ring *rx_ring, struct ice_rx_buf *rx_buf,  in ice_construct_skb()  argument
976 	skb = __napi_alloc_skb(&rx_ring->q_vector->napi, ICE_RX_HDR_SIZE,  in ice_construct_skb()
981 	skb_record_rx_queue(skb, rx_ring->q_index);  in ice_construct_skb()
997 		unsigned int truesize = ice_rx_pg_size(rx_ring) / 2;  in ice_construct_skb()
1025 ice_put_rx_buf(struct ice_ring *rx_ring, struct ice_rx_buf *rx_buf,  in ice_put_rx_buf()  argument
1028 	u16 ntc = rx_ring->next_to_clean + 1;  in ice_put_rx_buf()
1031 	ntc = (ntc < rx_ring->count) ? ntc : 0;  in ice_put_rx_buf()
1032 	rx_ring->next_to_clean = ntc;  in ice_put_rx_buf()
1039 		ice_reuse_rx_page(rx_ring, rx_buf);  in ice_put_rx_buf()
1042 		dma_unmap_page_attrs(rx_ring->dev, rx_buf->dma,  in ice_put_rx_buf()
1043 				     ice_rx_pg_size(rx_ring), DMA_FROM_DEVICE,  in ice_put_rx_buf()
1063 ice_is_non_eop(struct ice_ring *rx_ring, union ice_32b_rx_flex_desc *rx_desc,  in ice_is_non_eop()  argument
1072 	rx_ring->rx_buf[rx_ring->next_to_clean].skb = skb;  in ice_is_non_eop()
1073 	rx_ring->rx_stats.non_eop_descs++;  in ice_is_non_eop()
1090 int ice_clean_rx_irq(struct ice_ring *rx_ring, int budget)  in ice_clean_rx_irq()  argument
1093 	u16 cleaned_count = ICE_DESC_UNUSED(rx_ring);  in ice_clean_rx_irq()
1099 	xdp.rxq = &rx_ring->xdp_rxq;  in ice_clean_rx_irq()
1102 	xdp.frame_sz = ice_rx_frame_truesize(rx_ring, 0);  in ice_clean_rx_irq()
1117 		rx_desc = ICE_RX_DESC(rx_ring, rx_ring->next_to_clean);  in ice_clean_rx_irq()
1134 		if (rx_desc->wb.rxdid == FDIR_DESC_RXDID || !rx_ring->netdev) {  in ice_clean_rx_irq()
1135 			ice_put_rx_buf(rx_ring, NULL, 0);  in ice_clean_rx_irq()
1144 		rx_buf = ice_get_rx_buf(rx_ring, &skb, size, &rx_buf_pgcnt);  in ice_clean_rx_irq()
1155 		xdp.data_hard_start = xdp.data - ice_rx_offset(rx_ring);  in ice_clean_rx_irq()
1160 		xdp.frame_sz = ice_rx_frame_truesize(rx_ring, size);  in ice_clean_rx_irq()
1164 		xdp_prog = READ_ONCE(rx_ring->xdp_prog);  in ice_clean_rx_irq()
1170 		xdp_res = ice_run_xdp(rx_ring, &xdp, xdp_prog);  in ice_clean_rx_irq()
1184 		ice_put_rx_buf(rx_ring, rx_buf, rx_buf_pgcnt);  in ice_clean_rx_irq()
1188 			ice_add_rx_frag(rx_ring, rx_buf, skb, size);  in ice_clean_rx_irq()
1190 			if (ice_ring_uses_build_skb(rx_ring))  in ice_clean_rx_irq()
1191 				skb = ice_build_skb(rx_ring, rx_buf, &xdp);  in ice_clean_rx_irq()
1193 				skb = ice_construct_skb(rx_ring, rx_buf, &xdp);  in ice_clean_rx_irq()
1197 			rx_ring->rx_stats.alloc_buf_failed++;  in ice_clean_rx_irq()
1203 		ice_put_rx_buf(rx_ring, rx_buf, rx_buf_pgcnt);  in ice_clean_rx_irq()
1207 		if (ice_is_non_eop(rx_ring, rx_desc, skb))  in ice_clean_rx_irq()
1233 		ice_process_skb_fields(rx_ring, rx_desc, skb, rx_ptype);  in ice_clean_rx_irq()
1236 		ice_receive_skb(rx_ring, skb, vlan_tag);  in ice_clean_rx_irq()
1243 	failure = ice_alloc_rx_bufs(rx_ring, cleaned_count);  in ice_clean_rx_irq()
1246 		ice_finalize_xdp_rx(rx_ring, xdp_xmit);  in ice_clean_rx_irq()
1248 	ice_update_rx_ring_stats(rx_ring, total_rx_pkts, total_rx_bytes);  in ice_clean_rx_irq()