285 static int enetc_clean_rx_ring(struct enetc_bdr *rx_ring,
310 			  v->rx_ring.stats.packets,  in enetc_rx_net_dim()
311 			  v->rx_ring.stats.bytes,  in enetc_rx_net_dim()
328 	work_done = enetc_clean_rx_ring(&v->rx_ring, napi, budget);  in enetc_poll()
475 static bool enetc_new_page(struct enetc_bdr *rx_ring,  in enetc_new_page()  argument
485 	addr = dma_map_page(rx_ring->dev, page, 0, PAGE_SIZE, DMA_FROM_DEVICE);  in enetc_new_page()
486 	if (unlikely(dma_mapping_error(rx_ring->dev, addr))) {  in enetc_new_page()
499 static int enetc_refill_rx_ring(struct enetc_bdr *rx_ring, const int buff_cnt)  in enetc_refill_rx_ring()  argument
505 	i = rx_ring->next_to_use;  in enetc_refill_rx_ring()
506 	rx_swbd = &rx_ring->rx_swbd[i];  in enetc_refill_rx_ring()
507 	rxbd = enetc_rxbd(rx_ring, i);  in enetc_refill_rx_ring()
512 			if (unlikely(!enetc_new_page(rx_ring, rx_swbd))) {  in enetc_refill_rx_ring()
513 				rx_ring->stats.rx_alloc_errs++;  in enetc_refill_rx_ring()
524 		rxbd = enetc_rxbd_next(rx_ring, rxbd, i);  in enetc_refill_rx_ring()
527 		if (unlikely(i == rx_ring->bd_count)) {  in enetc_refill_rx_ring()
529 			rx_swbd = rx_ring->rx_swbd;  in enetc_refill_rx_ring()
534 		rx_ring->next_to_alloc = i; /* keep track from page reuse */  in enetc_refill_rx_ring()
535 		rx_ring->next_to_use = i;  in enetc_refill_rx_ring()
567 static void enetc_get_offloads(struct enetc_bdr *rx_ring,  in enetc_get_offloads()  argument
571 	struct enetc_ndev_priv *priv = netdev_priv(rx_ring->ndev);  in enetc_get_offloads()
574 	if (rx_ring->ndev->features & NETIF_F_RXCSUM) {  in enetc_get_offloads()
589 		enetc_get_rx_tstamp(rx_ring->ndev, rxbd, skb);  in enetc_get_offloads()
593 static void enetc_process_skb(struct enetc_bdr *rx_ring,  in enetc_process_skb()  argument
596 	skb_record_rx_queue(skb, rx_ring->index);  in enetc_process_skb()
597 	skb->protocol = eth_type_trans(skb, rx_ring->ndev);  in enetc_process_skb()
605 static void enetc_reuse_page(struct enetc_bdr *rx_ring,  in enetc_reuse_page()  argument
610 	new = &rx_ring->rx_swbd[rx_ring->next_to_alloc];  in enetc_reuse_page()
613 	enetc_bdr_idx_inc(rx_ring, &rx_ring->next_to_alloc);  in enetc_reuse_page()
619 static struct enetc_rx_swbd *enetc_get_rx_buff(struct enetc_bdr *rx_ring,  in enetc_get_rx_buff()  argument
622 	struct enetc_rx_swbd *rx_swbd = &rx_ring->rx_swbd[i];  in enetc_get_rx_buff()
624 	dma_sync_single_range_for_cpu(rx_ring->dev, rx_swbd->dma,  in enetc_get_rx_buff()
630 static void enetc_put_rx_buff(struct enetc_bdr *rx_ring,  in enetc_put_rx_buff()  argument
637 		enetc_reuse_page(rx_ring, rx_swbd);  in enetc_put_rx_buff()
640 		dma_sync_single_range_for_device(rx_ring->dev, rx_swbd->dma,  in enetc_put_rx_buff()
645 		dma_unmap_page(rx_ring->dev, rx_swbd->dma,  in enetc_put_rx_buff()
652 static struct sk_buff *enetc_map_rx_buff_to_skb(struct enetc_bdr *rx_ring,  in enetc_map_rx_buff_to_skb()  argument
655 	struct enetc_rx_swbd *rx_swbd = enetc_get_rx_buff(rx_ring, i, size);  in enetc_map_rx_buff_to_skb()
662 		rx_ring->stats.rx_alloc_errs++;  in enetc_map_rx_buff_to_skb()
669 	enetc_put_rx_buff(rx_ring, rx_swbd);  in enetc_map_rx_buff_to_skb()
674 static void enetc_add_rx_buff_to_skb(struct enetc_bdr *rx_ring, int i,  in enetc_add_rx_buff_to_skb()  argument
677 	struct enetc_rx_swbd *rx_swbd = enetc_get_rx_buff(rx_ring, i, size);  in enetc_add_rx_buff_to_skb()
682 	enetc_put_rx_buff(rx_ring, rx_swbd);  in enetc_add_rx_buff_to_skb()
687 static int enetc_clean_rx_ring(struct enetc_bdr *rx_ring,  in enetc_clean_rx_ring()  argument
693 	cleaned_cnt = enetc_bd_unused(rx_ring);  in enetc_clean_rx_ring()
695 	i = rx_ring->next_to_clean;  in enetc_clean_rx_ring()
706 			int count = enetc_refill_rx_ring(rx_ring, cleaned_cnt);  in enetc_clean_rx_ring()
709 			enetc_wr_reg_hot(rx_ring->rcir, rx_ring->next_to_use);  in enetc_clean_rx_ring()
713 		rxbd = enetc_rxbd(rx_ring, i);  in enetc_clean_rx_ring()
720 		enetc_wr_reg_hot(rx_ring->idr, BIT(rx_ring->index));  in enetc_clean_rx_ring()
723 		skb = enetc_map_rx_buff_to_skb(rx_ring, i, size);  in enetc_clean_rx_ring()
729 		enetc_get_offloads(rx_ring, rxbd, skb);  in enetc_clean_rx_ring()
733 		rxbd = enetc_rxbd_next(rx_ring, rxbd, i);  in enetc_clean_rx_ring()
734 		if (unlikely(++i == rx_ring->bd_count))  in enetc_clean_rx_ring()
745 				rxbd = enetc_rxbd_next(rx_ring, rxbd, i);  in enetc_clean_rx_ring()
746 				if (unlikely(++i == rx_ring->bd_count))  in enetc_clean_rx_ring()
750 			rx_ring->ndev->stats.rx_dropped++;  in enetc_clean_rx_ring()
751 			rx_ring->ndev->stats.rx_errors++;  in enetc_clean_rx_ring()
766 			enetc_add_rx_buff_to_skb(rx_ring, i, size, skb);  in enetc_clean_rx_ring()
770 			rxbd = enetc_rxbd_next(rx_ring, rxbd, i);  in enetc_clean_rx_ring()
771 			if (unlikely(++i == rx_ring->bd_count))  in enetc_clean_rx_ring()
777 		enetc_process_skb(rx_ring, skb);  in enetc_clean_rx_ring()
786 	rx_ring->next_to_clean = i;  in enetc_clean_rx_ring()
788 	rx_ring->stats.packets += rx_frm_cnt;  in enetc_clean_rx_ring()
789 	rx_ring->stats.bytes += rx_byte_cnt;  in enetc_clean_rx_ring()
952 		err = enetc_alloc_rxbdr(priv->rx_ring[i], extended);  in enetc_alloc_rx_resources()
962 		enetc_free_rxbdr(priv->rx_ring[i]);  in enetc_alloc_rx_resources()
972 		enetc_free_rxbdr(priv->rx_ring[i]);  in enetc_free_rx_resources()
992 static void enetc_free_rx_ring(struct enetc_bdr *rx_ring)  in enetc_free_rx_ring()  argument
996 	if (!rx_ring->rx_swbd)  in enetc_free_rx_ring()
999 	for (i = 0; i < rx_ring->bd_count; i++) {  in enetc_free_rx_ring()
1000 		struct enetc_rx_swbd *rx_swbd = &rx_ring->rx_swbd[i];  in enetc_free_rx_ring()
1005 		dma_unmap_page(rx_ring->dev, rx_swbd->dma,  in enetc_free_rx_ring()
1011 	rx_ring->next_to_clean = 0;  in enetc_free_rx_ring()
1012 	rx_ring->next_to_use = 0;  in enetc_free_rx_ring()
1013 	rx_ring->next_to_alloc = 0;  in enetc_free_rx_ring()
1021 		enetc_free_rx_ring(priv->rx_ring[i]);  in enetc_free_rxtx_rings()
1221 static void enetc_setup_rxbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring)  in enetc_setup_rxbdr()  argument
1223 	int idx = rx_ring->index;  in enetc_setup_rxbdr()
1227 		       lower_32_bits(rx_ring->bd_dma_base));  in enetc_setup_rxbdr()
1230 		       upper_32_bits(rx_ring->bd_dma_base));  in enetc_setup_rxbdr()
1232 	WARN_ON(!IS_ALIGNED(rx_ring->bd_count, 64)); /* multiple of 64 */  in enetc_setup_rxbdr()
1234 		       ENETC_RTBLENR_LEN(rx_ring->bd_count));  in enetc_setup_rxbdr()
1245 	if (rx_ring->ext_en)  in enetc_setup_rxbdr()
1248 	if (rx_ring->ndev->features & NETIF_F_HW_VLAN_CTAG_RX)  in enetc_setup_rxbdr()
1251 	rx_ring->rcir = hw->reg + ENETC_BDR(RX, idx, ENETC_RBCIR);  in enetc_setup_rxbdr()
1252 	rx_ring->idr = hw->reg + ENETC_SIRXIDR;  in enetc_setup_rxbdr()
1254 	enetc_refill_rx_ring(rx_ring, enetc_bd_unused(rx_ring));  in enetc_setup_rxbdr()
1255 	enetc_wr(hw, ENETC_SIRXIDR, rx_ring->next_to_use);  in enetc_setup_rxbdr()
1269 		enetc_setup_rxbdr(&priv->si->hw, priv->rx_ring[i]);  in enetc_setup_bdrs()
1272 static void enetc_clear_rxbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring)  in enetc_clear_rxbdr()  argument
1274 	int idx = rx_ring->index;  in enetc_clear_rxbdr()
1308 		enetc_clear_rxbdr(&priv->si->hw, priv->rx_ring[i]);  in enetc_clear_bdrs()
1634 		packets += priv->rx_ring[i]->stats.packets;  in enetc_get_stats()
1635 		bytes	+= priv->rx_ring[i]->stats.bytes;  in enetc_get_stats()
1870 		bdr = &v->rx_ring;  in enetc_alloc_msix()
1875 		priv->rx_ring[i] = bdr;  in enetc_alloc_msix()
1904 		priv->rx_ring[i] = NULL;  in enetc_free_msix()