603 static void tsnep_rx_ring_cleanup(struct tsnep_rx *rx)  in tsnep_rx_ring_cleanup()  argument
605 	struct device *dmadev = rx->adapter->dmadev;  in tsnep_rx_ring_cleanup()
610 		entry = &rx->entry[i];  in tsnep_rx_ring_cleanup()
612 			page_pool_put_full_page(rx->page_pool, entry->page,  in tsnep_rx_ring_cleanup()
617 	if (rx->page_pool)  in tsnep_rx_ring_cleanup()
618 		page_pool_destroy(rx->page_pool);  in tsnep_rx_ring_cleanup()
620 	memset(rx->entry, 0, sizeof(rx->entry));  in tsnep_rx_ring_cleanup()
623 		if (rx->page[i]) {  in tsnep_rx_ring_cleanup()
624 			dma_free_coherent(dmadev, PAGE_SIZE, rx->page[i],  in tsnep_rx_ring_cleanup()
625 					  rx->page_dma[i]);  in tsnep_rx_ring_cleanup()
626 			rx->page[i] = NULL;  in tsnep_rx_ring_cleanup()
627 			rx->page_dma[i] = 0;  in tsnep_rx_ring_cleanup()
632 static int tsnep_rx_alloc_buffer(struct tsnep_rx *rx,  in tsnep_rx_alloc_buffer()  argument
637 	page = page_pool_dev_alloc_pages(rx->page_pool);  in tsnep_rx_alloc_buffer()
644 	entry->desc->rx = __cpu_to_le64(entry->dma + TSNEP_SKB_PAD);  in tsnep_rx_alloc_buffer()
649 static int tsnep_rx_ring_init(struct tsnep_rx *rx)  in tsnep_rx_ring_init()  argument
651 	struct device *dmadev = rx->adapter->dmadev;  in tsnep_rx_ring_init()
659 		rx->page[i] =  in tsnep_rx_ring_init()
660 			dma_alloc_coherent(dmadev, PAGE_SIZE, &rx->page_dma[i],  in tsnep_rx_ring_init()
662 		if (!rx->page[i]) {  in tsnep_rx_ring_init()
667 			entry = &rx->entry[TSNEP_RING_ENTRIES_PER_PAGE * i + j];  in tsnep_rx_ring_init()
669 				(((u8 *)rx->page[i]) + TSNEP_DESC_SIZE * j);  in tsnep_rx_ring_init()
672 			entry->desc_dma = rx->page_dma[i] + TSNEP_DESC_SIZE * j;  in tsnep_rx_ring_init()
684 	rx->page_pool = page_pool_create(&pp_params);  in tsnep_rx_ring_init()
685 	if (IS_ERR(rx->page_pool)) {  in tsnep_rx_ring_init()
686 		retval = PTR_ERR(rx->page_pool);  in tsnep_rx_ring_init()
687 		rx->page_pool = NULL;  in tsnep_rx_ring_init()
692 		entry = &rx->entry[i];  in tsnep_rx_ring_init()
693 		next_entry = &rx->entry[(i + 1) % TSNEP_RING_SIZE];  in tsnep_rx_ring_init()
696 		retval = tsnep_rx_alloc_buffer(rx, entry);  in tsnep_rx_ring_init()
704 	tsnep_rx_ring_cleanup(rx);  in tsnep_rx_ring_init()
708 static void tsnep_rx_activate(struct tsnep_rx *rx, int index)  in tsnep_rx_activate()  argument
710 	struct tsnep_rx_entry *entry = &rx->entry[index];  in tsnep_rx_activate()
715 	if (index == rx->increment_owner_counter) {  in tsnep_rx_activate()
716 		rx->owner_counter++;  in tsnep_rx_activate()
717 		if (rx->owner_counter == 4)  in tsnep_rx_activate()
718 			rx->owner_counter = 1;  in tsnep_rx_activate()
719 		rx->increment_owner_counter--;  in tsnep_rx_activate()
720 		if (rx->increment_owner_counter < 0)  in tsnep_rx_activate()
721 			rx->increment_owner_counter = TSNEP_RING_SIZE - 1;  in tsnep_rx_activate()
724 		(rx->owner_counter << TSNEP_DESC_OWNER_COUNTER_SHIFT) &  in tsnep_rx_activate()
735 static struct sk_buff *tsnep_build_skb(struct tsnep_rx *rx, struct page *page,  in tsnep_build_skb()  argument
748 	if (rx->adapter->hwtstamp_config.rx_filter == HWTSTAMP_FILTER_ALL) {  in tsnep_build_skb()
760 	skb_record_rx_queue(skb, rx->queue_index);  in tsnep_build_skb()
761 	skb->protocol = eth_type_trans(skb, rx->adapter->netdev);  in tsnep_build_skb()
766 static int tsnep_rx_poll(struct tsnep_rx *rx, struct napi_struct *napi,  in tsnep_rx_poll()  argument
769 	struct device *dmadev = rx->adapter->dmadev;  in tsnep_rx_poll()
779 	dma_dir = page_pool_get_dma_dir(rx->page_pool);  in tsnep_rx_poll()
782 		entry = &rx->entry[rx->read];  in tsnep_rx_poll()
803 		retval = tsnep_rx_alloc_buffer(rx, entry);  in tsnep_rx_poll()
805 			skb = tsnep_build_skb(rx, page, length);  in tsnep_rx_poll()
807 				page_pool_release_page(rx->page_pool, page);  in tsnep_rx_poll()
809 				rx->packets++;  in tsnep_rx_poll()
810 				rx->bytes += length -  in tsnep_rx_poll()
813 					rx->multicast++;  in tsnep_rx_poll()
817 				page_pool_recycle_direct(rx->page_pool, page);  in tsnep_rx_poll()
819 				rx->dropped++;  in tsnep_rx_poll()
823 			rx->dropped++;  in tsnep_rx_poll()
826 		tsnep_rx_activate(rx, rx->read);  in tsnep_rx_poll()
830 		rx->read = (rx->read + 1) % TSNEP_RING_SIZE;  in tsnep_rx_poll()
839 		iowrite32(TSNEP_CONTROL_RX_ENABLE, rx->addr + TSNEP_CONTROL);  in tsnep_rx_poll()
845 static bool tsnep_rx_pending(struct tsnep_rx *rx)  in tsnep_rx_pending()  argument
849 	entry = &rx->entry[rx->read];  in tsnep_rx_pending()
859 			 int queue_index, struct tsnep_rx *rx)  in tsnep_rx_open()  argument
865 	memset(rx, 0, sizeof(*rx));  in tsnep_rx_open()
866 	rx->adapter = adapter;  in tsnep_rx_open()
867 	rx->addr = addr;  in tsnep_rx_open()
868 	rx->queue_index = queue_index;  in tsnep_rx_open()
870 	retval = tsnep_rx_ring_init(rx);  in tsnep_rx_open()
874 	dma = rx->entry[0].desc_dma | TSNEP_RESET_OWNER_COUNTER;  in tsnep_rx_open()
875 	iowrite32(DMA_ADDR_LOW(dma), rx->addr + TSNEP_RX_DESC_ADDR_LOW);  in tsnep_rx_open()
876 	iowrite32(DMA_ADDR_HIGH(dma), rx->addr + TSNEP_RX_DESC_ADDR_HIGH);  in tsnep_rx_open()
877 	rx->owner_counter = 1;  in tsnep_rx_open()
878 	rx->increment_owner_counter = TSNEP_RING_SIZE - 1;  in tsnep_rx_open()
881 		tsnep_rx_activate(rx, i);  in tsnep_rx_open()
886 	iowrite32(TSNEP_CONTROL_RX_ENABLE, rx->addr + TSNEP_CONTROL);  in tsnep_rx_open()
891 static void tsnep_rx_close(struct tsnep_rx *rx)  in tsnep_rx_close()  argument
895 	iowrite32(TSNEP_CONTROL_RX_DISABLE, rx->addr + TSNEP_CONTROL);  in tsnep_rx_close()
896 	readx_poll_timeout(ioread32, rx->addr + TSNEP_CONTROL, val,  in tsnep_rx_close()
900 	tsnep_rx_ring_cleanup(rx);  in tsnep_rx_close()
908 	if (queue->rx && tsnep_rx_pending(queue->rx))  in tsnep_pending()
924 	if (queue->rx) {  in tsnep_poll()
925 		done = tsnep_rx_poll(queue->rx, napi, budget);  in tsnep_poll()
962 		if (queue->tx && queue->rx)  in tsnep_request_irq()
964 				queue->rx->queue_index);  in tsnep_request_irq()
970 				queue->rx->queue_index);  in tsnep_request_irq()
1019 		if (adapter->queue[i].rx) {  in tsnep_netdev_open()
1023 					       adapter->queue[i].rx);  in tsnep_netdev_open()
1069 		if (adapter->queue[i].rx)  in tsnep_netdev_open()
1070 			tsnep_rx_close(adapter->queue[i].rx);  in tsnep_netdev_open()
1093 		if (adapter->queue[i].rx)  in tsnep_netdev_close()
1094 			tsnep_rx_close(adapter->queue[i].rx);  in tsnep_netdev_close()
1154 		stats->rx_packets += adapter->rx[i].packets;  in tsnep_netdev_get_stats64()
1155 		stats->rx_bytes += adapter->rx[i].bytes;  in tsnep_netdev_get_stats64()
1156 		stats->rx_dropped += adapter->rx[i].dropped;  in tsnep_netdev_get_stats64()
1157 		stats->multicast += adapter->rx[i].multicast;  in tsnep_netdev_get_stats64()
1372 	adapter->queue[0].rx = &adapter->rx[0];  in tsnep_queue_init()
1391 		adapter->queue[i].rx = &adapter->rx[i];  in tsnep_queue_init()