375 	if (!rx_ring->rx_buf)  in ice_clean_rx_ring()
385 		struct ice_rx_buf *rx_buf = &rx_ring->rx_buf[i];  in ice_clean_rx_ring()  local
387 		if (rx_buf->skb) {  in ice_clean_rx_ring()
388 			dev_kfree_skb(rx_buf->skb);  in ice_clean_rx_ring()
389 			rx_buf->skb = NULL;  in ice_clean_rx_ring()
391 		if (!rx_buf->page)  in ice_clean_rx_ring()
397 		dma_sync_single_range_for_cpu(dev, rx_buf->dma,  in ice_clean_rx_ring()
398 					      rx_buf->page_offset,  in ice_clean_rx_ring()
403 		dma_unmap_page_attrs(dev, rx_buf->dma, ice_rx_pg_size(rx_ring),  in ice_clean_rx_ring()
405 		__page_frag_cache_drain(rx_buf->page, rx_buf->pagecnt_bias);  in ice_clean_rx_ring()
407 		rx_buf->page = NULL;  in ice_clean_rx_ring()
408 		rx_buf->page_offset = 0;  in ice_clean_rx_ring()
412 	memset(rx_ring->rx_buf, 0, sizeof(*rx_ring->rx_buf) * rx_ring->count);  in ice_clean_rx_ring()
435 	devm_kfree(rx_ring->dev, rx_ring->rx_buf);  in ice_free_rx_ring()
436 	rx_ring->rx_buf = NULL;  in ice_free_rx_ring()
459 	WARN_ON(rx_ring->rx_buf);  in ice_setup_rx_ring()
460 	rx_ring->rx_buf =  in ice_setup_rx_ring()
461 		devm_kzalloc(dev, sizeof(*rx_ring->rx_buf) * rx_ring->count,  in ice_setup_rx_ring()
463 	if (!rx_ring->rx_buf)  in ice_setup_rx_ring()
491 	devm_kfree(dev, rx_ring->rx_buf);  in ice_setup_rx_ring()
492 	rx_ring->rx_buf = NULL;  in ice_setup_rx_ring()
692 	bi = &rx_ring->rx_buf[ntu];  in ice_alloc_rx_bufs()
715 			bi = rx_ring->rx_buf;  in ice_alloc_rx_bufs()
751 ice_rx_buf_adjust_pg_offset(struct ice_rx_buf *rx_buf, unsigned int size)  in ice_rx_buf_adjust_pg_offset()  argument
755 	rx_buf->page_offset ^= size;  in ice_rx_buf_adjust_pg_offset()
758 	rx_buf->page_offset += size;  in ice_rx_buf_adjust_pg_offset()
773 ice_can_reuse_rx_page(struct ice_rx_buf *rx_buf, int rx_buf_pgcnt)  in ice_can_reuse_rx_page()  argument
775 	unsigned int pagecnt_bias = rx_buf->pagecnt_bias;  in ice_can_reuse_rx_page()
776 	struct page *page = rx_buf->page;  in ice_can_reuse_rx_page()
789 	if (rx_buf->page_offset > ICE_LAST_OFFSET)  in ice_can_reuse_rx_page()
799 		rx_buf->pagecnt_bias = USHRT_MAX;  in ice_can_reuse_rx_page()
817 ice_add_rx_frag(struct ice_ring *rx_ring, struct ice_rx_buf *rx_buf,  in ice_add_rx_frag()  argument
828 	skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_buf->page,  in ice_add_rx_frag()
829 			rx_buf->page_offset, size, truesize);  in ice_add_rx_frag()
832 	ice_rx_buf_adjust_pg_offset(rx_buf, truesize);  in ice_add_rx_frag()
848 	new_buf = &rx_ring->rx_buf[nta];  in ice_reuse_rx_page()
878 	struct ice_rx_buf *rx_buf;  in ice_get_rx_buf()  local
880 	rx_buf = &rx_ring->rx_buf[rx_ring->next_to_clean];  in ice_get_rx_buf()
883 		page_count(rx_buf->page);  in ice_get_rx_buf()
887 	prefetchw(rx_buf->page);  in ice_get_rx_buf()
888 	*skb = rx_buf->skb;  in ice_get_rx_buf()
891 		return rx_buf;  in ice_get_rx_buf()
893 	dma_sync_single_range_for_cpu(rx_ring->dev, rx_buf->dma,  in ice_get_rx_buf()
894 				      rx_buf->page_offset, size,  in ice_get_rx_buf()
898 	rx_buf->pagecnt_bias--;  in ice_get_rx_buf()
900 	return rx_buf;  in ice_get_rx_buf()
913 ice_build_skb(struct ice_ring *rx_ring, struct ice_rx_buf *rx_buf,  in ice_build_skb()  argument
949 	ice_rx_buf_adjust_pg_offset(rx_buf, truesize);  in ice_build_skb()
965 ice_construct_skb(struct ice_ring *rx_ring, struct ice_rx_buf *rx_buf,  in ice_construct_skb()  argument
999 		skb_add_rx_frag(skb, 0, rx_buf->page,  in ice_construct_skb()
1000 				rx_buf->page_offset + headlen, size, truesize);  in ice_construct_skb()
1002 		ice_rx_buf_adjust_pg_offset(rx_buf, truesize);  in ice_construct_skb()
1008 		rx_buf->pagecnt_bias++;  in ice_construct_skb()
1025 ice_put_rx_buf(struct ice_ring *rx_ring, struct ice_rx_buf *rx_buf,  in ice_put_rx_buf()  argument
1034 	if (!rx_buf)  in ice_put_rx_buf()
1037 	if (ice_can_reuse_rx_page(rx_buf, rx_buf_pgcnt)) {  in ice_put_rx_buf()
1039 		ice_reuse_rx_page(rx_ring, rx_buf);  in ice_put_rx_buf()
1042 		dma_unmap_page_attrs(rx_ring->dev, rx_buf->dma,  in ice_put_rx_buf()
1045 		__page_frag_cache_drain(rx_buf->page, rx_buf->pagecnt_bias);  in ice_put_rx_buf()
1049 	rx_buf->page = NULL;  in ice_put_rx_buf()
1050 	rx_buf->skb = NULL;  in ice_put_rx_buf()
1072 	rx_ring->rx_buf[rx_ring->next_to_clean].skb = skb;  in ice_is_non_eop()
1108 		struct ice_rx_buf *rx_buf;  in ice_clean_rx_irq()  local
1144 		rx_buf = ice_get_rx_buf(rx_ring, &skb, size, &rx_buf_pgcnt);  in ice_clean_rx_irq()
1154 		xdp.data = page_address(rx_buf->page) + rx_buf->page_offset;  in ice_clean_rx_irq()
1176 			ice_rx_buf_adjust_pg_offset(rx_buf, xdp.frame_sz);  in ice_clean_rx_irq()
1178 			rx_buf->pagecnt_bias++;  in ice_clean_rx_irq()
1184 		ice_put_rx_buf(rx_ring, rx_buf, rx_buf_pgcnt);  in ice_clean_rx_irq()
1188 			ice_add_rx_frag(rx_ring, rx_buf, skb, size);  in ice_clean_rx_irq()
1191 				skb = ice_build_skb(rx_ring, rx_buf, &xdp);  in ice_clean_rx_irq()
1193 				skb = ice_construct_skb(rx_ring, rx_buf, &xdp);  in ice_clean_rx_irq()
1198 			if (rx_buf)  in ice_clean_rx_irq()
1199 				rx_buf->pagecnt_bias++;  in ice_clean_rx_irq()
1203 		ice_put_rx_buf(rx_ring, rx_buf, rx_buf_pgcnt);  in ice_clean_rx_irq()