343 static void igc_clean_rx_ring(struct igc_ring *rx_ring)  in igc_clean_rx_ring()  argument
345 	u16 i = rx_ring->next_to_clean;  in igc_clean_rx_ring()
347 	dev_kfree_skb(rx_ring->skb);  in igc_clean_rx_ring()
348 	rx_ring->skb = NULL;  in igc_clean_rx_ring()
351 	while (i != rx_ring->next_to_alloc) {  in igc_clean_rx_ring()
352 		struct igc_rx_buffer *buffer_info = &rx_ring->rx_buffer_info[i];  in igc_clean_rx_ring()
357 		dma_sync_single_range_for_cpu(rx_ring->dev,  in igc_clean_rx_ring()
360 					      igc_rx_bufsz(rx_ring),  in igc_clean_rx_ring()
364 		dma_unmap_page_attrs(rx_ring->dev,  in igc_clean_rx_ring()
366 				     igc_rx_pg_size(rx_ring),  in igc_clean_rx_ring()
373 		if (i == rx_ring->count)  in igc_clean_rx_ring()
377 	rx_ring->next_to_alloc = 0;  in igc_clean_rx_ring()
378 	rx_ring->next_to_clean = 0;  in igc_clean_rx_ring()
379 	rx_ring->next_to_use = 0;  in igc_clean_rx_ring()
391 		if (adapter->rx_ring[i])  in igc_clean_all_rx_rings()
392 			igc_clean_rx_ring(adapter->rx_ring[i]);  in igc_clean_all_rx_rings()
401 void igc_free_rx_resources(struct igc_ring *rx_ring)  in igc_free_rx_resources()  argument
403 	igc_clean_rx_ring(rx_ring);  in igc_free_rx_resources()
405 	vfree(rx_ring->rx_buffer_info);  in igc_free_rx_resources()
406 	rx_ring->rx_buffer_info = NULL;  in igc_free_rx_resources()
409 	if (!rx_ring->desc)  in igc_free_rx_resources()
412 	dma_free_coherent(rx_ring->dev, rx_ring->size,  in igc_free_rx_resources()
413 			  rx_ring->desc, rx_ring->dma);  in igc_free_rx_resources()
415 	rx_ring->desc = NULL;  in igc_free_rx_resources()
429 		igc_free_rx_resources(adapter->rx_ring[i]);  in igc_free_all_rx_resources()
438 int igc_setup_rx_resources(struct igc_ring *rx_ring)  in igc_setup_rx_resources()  argument
440 	struct net_device *ndev = rx_ring->netdev;  in igc_setup_rx_resources()
441 	struct device *dev = rx_ring->dev;  in igc_setup_rx_resources()
444 	size = sizeof(struct igc_rx_buffer) * rx_ring->count;  in igc_setup_rx_resources()
445 	rx_ring->rx_buffer_info = vzalloc(size);  in igc_setup_rx_resources()
446 	if (!rx_ring->rx_buffer_info)  in igc_setup_rx_resources()
452 	rx_ring->size = rx_ring->count * desc_len;  in igc_setup_rx_resources()
453 	rx_ring->size = ALIGN(rx_ring->size, 4096);  in igc_setup_rx_resources()
455 	rx_ring->desc = dma_alloc_coherent(dev, rx_ring->size,  in igc_setup_rx_resources()
456 					   &rx_ring->dma, GFP_KERNEL);  in igc_setup_rx_resources()
458 	if (!rx_ring->desc)  in igc_setup_rx_resources()
461 	rx_ring->next_to_alloc = 0;  in igc_setup_rx_resources()
462 	rx_ring->next_to_clean = 0;  in igc_setup_rx_resources()
463 	rx_ring->next_to_use = 0;  in igc_setup_rx_resources()
468 	vfree(rx_ring->rx_buffer_info);  in igc_setup_rx_resources()
469 	rx_ring->rx_buffer_info = NULL;  in igc_setup_rx_resources()
487 		err = igc_setup_rx_resources(adapter->rx_ring[i]);  in igc_setup_all_rx_resources()
491 				igc_free_rx_resources(adapter->rx_ring[i]);  in igc_setup_all_rx_resources()
576 		igc_configure_rx_ring(adapter, adapter->rx_ring[i]);  in igc_configure_rx()
1480 static void igc_process_skb_fields(struct igc_ring *rx_ring,  in igc_process_skb_fields()  argument
1484 	igc_rx_hash(rx_ring, rx_desc, skb);  in igc_process_skb_fields()
1486 	igc_rx_checksum(rx_ring, rx_desc, skb);  in igc_process_skb_fields()
1488 	skb_record_rx_queue(skb, rx_ring->queue_index);  in igc_process_skb_fields()
1490 	skb->protocol = eth_type_trans(skb, rx_ring->netdev);  in igc_process_skb_fields()
1493 static struct igc_rx_buffer *igc_get_rx_buffer(struct igc_ring *rx_ring,  in igc_get_rx_buffer()  argument
1498 	rx_buffer = &rx_ring->rx_buffer_info[rx_ring->next_to_clean];  in igc_get_rx_buffer()
1502 	dma_sync_single_range_for_cpu(rx_ring->dev,  in igc_get_rx_buffer()
1522 static void igc_add_rx_frag(struct igc_ring *rx_ring,  in igc_add_rx_frag()  argument
1528 	unsigned int truesize = igc_rx_pg_size(rx_ring) / 2;  in igc_add_rx_frag()
1534 	unsigned int truesize = ring_uses_build_skb(rx_ring) ?  in igc_add_rx_frag()
1543 static struct sk_buff *igc_build_skb(struct igc_ring *rx_ring,  in igc_build_skb()  argument
1550 	unsigned int truesize = igc_rx_pg_size(rx_ring) / 2;  in igc_build_skb()
1579 static struct sk_buff *igc_construct_skb(struct igc_ring *rx_ring,  in igc_construct_skb()  argument
1586 	unsigned int truesize = igc_rx_pg_size(rx_ring) / 2;  in igc_construct_skb()
1597 	skb = napi_alloc_skb(&rx_ring->q_vector->napi, IGC_RX_HDR_LEN);  in igc_construct_skb()
1602 		igc_ptp_rx_pktstamp(rx_ring->q_vector, va, skb);  in igc_construct_skb()
1640 static void igc_reuse_rx_page(struct igc_ring *rx_ring,  in igc_reuse_rx_page()  argument
1643 	u16 nta = rx_ring->next_to_alloc;  in igc_reuse_rx_page()
1646 	new_buff = &rx_ring->rx_buffer_info[nta];  in igc_reuse_rx_page()
1650 	rx_ring->next_to_alloc = (nta < rx_ring->count) ? nta : 0;  in igc_reuse_rx_page()
1710 static bool igc_is_non_eop(struct igc_ring *rx_ring,  in igc_is_non_eop()  argument
1713 	u32 ntc = rx_ring->next_to_clean + 1;  in igc_is_non_eop()
1716 	ntc = (ntc < rx_ring->count) ? ntc : 0;  in igc_is_non_eop()
1717 	rx_ring->next_to_clean = ntc;  in igc_is_non_eop()
1719 	prefetch(IGC_RX_DESC(rx_ring, ntc));  in igc_is_non_eop()
1741 static bool igc_cleanup_headers(struct igc_ring *rx_ring,  in igc_cleanup_headers()  argument
1746 		struct net_device *netdev = rx_ring->netdev;  in igc_cleanup_headers()
1761 static void igc_put_rx_buffer(struct igc_ring *rx_ring,  in igc_put_rx_buffer()  argument
1766 		igc_reuse_rx_page(rx_ring, rx_buffer);  in igc_put_rx_buffer()
1771 		dma_unmap_page_attrs(rx_ring->dev, rx_buffer->dma,  in igc_put_rx_buffer()
1772 				     igc_rx_pg_size(rx_ring), DMA_FROM_DEVICE,  in igc_put_rx_buffer()
1782 static inline unsigned int igc_rx_offset(struct igc_ring *rx_ring)  in igc_rx_offset()  argument
1784 	return ring_uses_build_skb(rx_ring) ? IGC_SKB_PAD : 0;  in igc_rx_offset()
1787 static bool igc_alloc_mapped_page(struct igc_ring *rx_ring,  in igc_alloc_mapped_page()  argument
1798 	page = dev_alloc_pages(igc_rx_pg_order(rx_ring));  in igc_alloc_mapped_page()
1800 		rx_ring->rx_stats.alloc_failed++;  in igc_alloc_mapped_page()
1805 	dma = dma_map_page_attrs(rx_ring->dev, page, 0,  in igc_alloc_mapped_page()
1806 				 igc_rx_pg_size(rx_ring),  in igc_alloc_mapped_page()
1813 	if (dma_mapping_error(rx_ring->dev, dma)) {  in igc_alloc_mapped_page()
1816 		rx_ring->rx_stats.alloc_failed++;  in igc_alloc_mapped_page()
1822 	bi->page_offset = igc_rx_offset(rx_ring);  in igc_alloc_mapped_page()
1833 static void igc_alloc_rx_buffers(struct igc_ring *rx_ring, u16 cleaned_count)  in igc_alloc_rx_buffers()  argument
1836 	u16 i = rx_ring->next_to_use;  in igc_alloc_rx_buffers()
1844 	rx_desc = IGC_RX_DESC(rx_ring, i);  in igc_alloc_rx_buffers()
1845 	bi = &rx_ring->rx_buffer_info[i];  in igc_alloc_rx_buffers()
1846 	i -= rx_ring->count;  in igc_alloc_rx_buffers()
1848 	bufsz = igc_rx_bufsz(rx_ring);  in igc_alloc_rx_buffers()
1851 		if (!igc_alloc_mapped_page(rx_ring, bi))  in igc_alloc_rx_buffers()
1855 		dma_sync_single_range_for_device(rx_ring->dev, bi->dma,  in igc_alloc_rx_buffers()
1868 			rx_desc = IGC_RX_DESC(rx_ring, 0);  in igc_alloc_rx_buffers()
1869 			bi = rx_ring->rx_buffer_info;  in igc_alloc_rx_buffers()
1870 			i -= rx_ring->count;  in igc_alloc_rx_buffers()
1879 	i += rx_ring->count;  in igc_alloc_rx_buffers()
1881 	if (rx_ring->next_to_use != i) {  in igc_alloc_rx_buffers()
1883 		rx_ring->next_to_use = i;  in igc_alloc_rx_buffers()
1886 		rx_ring->next_to_alloc = i;  in igc_alloc_rx_buffers()
1894 		writel(i, rx_ring->tail);  in igc_alloc_rx_buffers()
1901 	struct igc_ring *rx_ring = q_vector->rx.ring;  in igc_clean_rx_irq()  local
1902 	struct sk_buff *skb = rx_ring->skb;  in igc_clean_rx_irq()
1903 	u16 cleaned_count = igc_desc_unused(rx_ring);  in igc_clean_rx_irq()
1912 			igc_alloc_rx_buffers(rx_ring, cleaned_count);  in igc_clean_rx_irq()
1916 		rx_desc = IGC_RX_DESC(rx_ring, rx_ring->next_to_clean);  in igc_clean_rx_irq()
1927 		rx_buffer = igc_get_rx_buffer(rx_ring, size);  in igc_clean_rx_irq()
1931 			igc_add_rx_frag(rx_ring, rx_buffer, skb, size);  in igc_clean_rx_irq()
1932 		else if (ring_uses_build_skb(rx_ring))  in igc_clean_rx_irq()
1933 			skb = igc_build_skb(rx_ring, rx_buffer, rx_desc, size);  in igc_clean_rx_irq()
1935 			skb = igc_construct_skb(rx_ring, rx_buffer,  in igc_clean_rx_irq()
1940 			rx_ring->rx_stats.alloc_failed++;  in igc_clean_rx_irq()
1945 		igc_put_rx_buffer(rx_ring, rx_buffer);  in igc_clean_rx_irq()
1949 		if (igc_is_non_eop(rx_ring, rx_desc))  in igc_clean_rx_irq()
1953 		if (igc_cleanup_headers(rx_ring, rx_desc, skb)) {  in igc_clean_rx_irq()
1962 		igc_process_skb_fields(rx_ring, rx_desc, skb);  in igc_clean_rx_irq()
1974 	rx_ring->skb = skb;  in igc_clean_rx_irq()
1976 	u64_stats_update_begin(&rx_ring->rx_syncp);  in igc_clean_rx_irq()
1977 	rx_ring->rx_stats.packets += total_packets;  in igc_clean_rx_irq()
1978 	rx_ring->rx_stats.bytes += total_bytes;  in igc_clean_rx_irq()
1979 	u64_stats_update_end(&rx_ring->rx_syncp);  in igc_clean_rx_irq()
1984 		igc_alloc_rx_buffers(rx_ring, cleaned_count);  in igc_clean_rx_irq()
2672 		struct igc_ring *ring = adapter->rx_ring[i];  in igc_configure()
2889 		adapter->rx_ring[q_vector->rx.ring->queue_index] = NULL;  in igc_reset_q_vector()
3272 			adapter->rx_ring[i]->reg_idx = i;  in igc_cache_ring_register()
3420 		adapter->rx_ring[rxr_idx] = ring;  in igc_alloc_q_vector()
3631 		struct igc_ring *ring = adapter->rx_ring[i];  in igc_update_stats()