273 	if (!rx_ring->rx_buf)  in ice_clean_rx_ring()
278 		struct ice_rx_buf *rx_buf = &rx_ring->rx_buf[i];  in ice_clean_rx_ring()  local
280 		if (rx_buf->skb) {  in ice_clean_rx_ring()
281 			dev_kfree_skb(rx_buf->skb);  in ice_clean_rx_ring()
282 			rx_buf->skb = NULL;  in ice_clean_rx_ring()
284 		if (!rx_buf->page)  in ice_clean_rx_ring()
290 		dma_sync_single_range_for_cpu(dev, rx_buf->dma,  in ice_clean_rx_ring()
291 					      rx_buf->page_offset,  in ice_clean_rx_ring()
295 		dma_unmap_page_attrs(dev, rx_buf->dma, PAGE_SIZE,  in ice_clean_rx_ring()
297 		__page_frag_cache_drain(rx_buf->page, rx_buf->pagecnt_bias);  in ice_clean_rx_ring()
299 		rx_buf->page = NULL;  in ice_clean_rx_ring()
300 		rx_buf->page_offset = 0;  in ice_clean_rx_ring()
303 	memset(rx_ring->rx_buf, 0, sizeof(*rx_ring->rx_buf) * rx_ring->count);  in ice_clean_rx_ring()
322 	devm_kfree(rx_ring->dev, rx_ring->rx_buf);  in ice_free_rx_ring()
323 	rx_ring->rx_buf = NULL;  in ice_free_rx_ring()
346 	WARN_ON(rx_ring->rx_buf);  in ice_setup_rx_ring()
347 	rx_ring->rx_buf =  in ice_setup_rx_ring()
348 		devm_kzalloc(dev, sizeof(*rx_ring->rx_buf) * rx_ring->count,  in ice_setup_rx_ring()
350 	if (!rx_ring->rx_buf)  in ice_setup_rx_ring()
369 	devm_kfree(dev, rx_ring->rx_buf);  in ice_setup_rx_ring()
370 	rx_ring->rx_buf = NULL;  in ice_setup_rx_ring()
479 	bi = &rx_ring->rx_buf[ntu];  in ice_alloc_rx_bufs()
502 			bi = rx_ring->rx_buf;  in ice_alloc_rx_bufs()
538 ice_rx_buf_adjust_pg_offset(struct ice_rx_buf *rx_buf, unsigned int size)  in ice_rx_buf_adjust_pg_offset()  argument
542 	rx_buf->page_offset ^= size;  in ice_rx_buf_adjust_pg_offset()
545 	rx_buf->page_offset += size;  in ice_rx_buf_adjust_pg_offset()
558 static bool ice_can_reuse_rx_page(struct ice_rx_buf *rx_buf)  in ice_can_reuse_rx_page()  argument
563 	unsigned int pagecnt_bias = rx_buf->pagecnt_bias;  in ice_can_reuse_rx_page()
564 	struct page *page = rx_buf->page;  in ice_can_reuse_rx_page()
575 	if (rx_buf->page_offset > last_offset)  in ice_can_reuse_rx_page()
585 		rx_buf->pagecnt_bias = USHRT_MAX;  in ice_can_reuse_rx_page()
602 ice_add_rx_frag(struct ice_rx_buf *rx_buf, struct sk_buff *skb,  in ice_add_rx_frag()  argument
613 	skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_buf->page,  in ice_add_rx_frag()
614 			rx_buf->page_offset, size, truesize);  in ice_add_rx_frag()
617 	ice_rx_buf_adjust_pg_offset(rx_buf, truesize);  in ice_add_rx_frag()
633 	new_buf = &rx_ring->rx_buf[nta];  in ice_reuse_rx_page()
662 	struct ice_rx_buf *rx_buf;  in ice_get_rx_buf()  local
664 	rx_buf = &rx_ring->rx_buf[rx_ring->next_to_clean];  in ice_get_rx_buf()
665 	prefetchw(rx_buf->page);  in ice_get_rx_buf()
666 	*skb = rx_buf->skb;  in ice_get_rx_buf()
669 		return rx_buf;  in ice_get_rx_buf()
671 	dma_sync_single_range_for_cpu(rx_ring->dev, rx_buf->dma,  in ice_get_rx_buf()
672 				      rx_buf->page_offset, size,  in ice_get_rx_buf()
676 	rx_buf->pagecnt_bias--;  in ice_get_rx_buf()
678 	return rx_buf;  in ice_get_rx_buf()
692 ice_construct_skb(struct ice_ring *rx_ring, struct ice_rx_buf *rx_buf,  in ice_construct_skb()  argument
695 	void *va = page_address(rx_buf->page) + rx_buf->page_offset;  in ice_construct_skb()
728 		skb_add_rx_frag(skb, 0, rx_buf->page,  in ice_construct_skb()
729 				rx_buf->page_offset + headlen, size, truesize);  in ice_construct_skb()
731 		ice_rx_buf_adjust_pg_offset(rx_buf, truesize);  in ice_construct_skb()
737 		rx_buf->pagecnt_bias++;  in ice_construct_skb()
751 static void ice_put_rx_buf(struct ice_ring *rx_ring, struct ice_rx_buf *rx_buf)  in ice_put_rx_buf()  argument
753 	if (!rx_buf)  in ice_put_rx_buf()
756 	if (ice_can_reuse_rx_page(rx_buf)) {  in ice_put_rx_buf()
758 		ice_reuse_rx_page(rx_ring, rx_buf);  in ice_put_rx_buf()
762 		dma_unmap_page_attrs(rx_ring->dev, rx_buf->dma, PAGE_SIZE,  in ice_put_rx_buf()
764 		__page_frag_cache_drain(rx_buf->page, rx_buf->pagecnt_bias);  in ice_put_rx_buf()
768 	rx_buf->page = NULL;  in ice_put_rx_buf()
769 	rx_buf->skb = NULL;  in ice_put_rx_buf()
839 	rx_ring->rx_buf[ntc].skb = skb;  in ice_is_non_eop()
1014 		struct ice_rx_buf *rx_buf;  in ice_clean_rx_irq()  local
1043 		rx_buf = ice_get_rx_buf(rx_ring, &skb, size);  in ice_clean_rx_irq()
1046 			ice_add_rx_frag(rx_buf, skb, size);  in ice_clean_rx_irq()
1048 			skb = ice_construct_skb(rx_ring, rx_buf, size);  in ice_clean_rx_irq()
1053 			if (rx_buf)  in ice_clean_rx_irq()
1054 				rx_buf->pagecnt_bias++;  in ice_clean_rx_irq()
1058 		ice_put_rx_buf(rx_ring, rx_buf);  in ice_clean_rx_irq()