344 	struct virtnet_info *vi = vq->vdev->priv;  in skb_xmit_done()  local
345 	struct napi_struct *napi = &vi->sq[vq2txq(vq)].napi;  in skb_xmit_done()
354 		netif_wake_subqueue(vi->dev, vq2txq(vq));  in skb_xmit_done()
375 static struct sk_buff *page_to_skb(struct virtnet_info *vi,  in page_to_skb()  argument
395 	hdr_len = vi->hdr_len;  in page_to_skb()
396 	if (vi->mergeable_rx_bufs)  in page_to_skb()
422 	if (vi->mergeable_rx_bufs) {  in page_to_skb()
457 static int __virtnet_xdp_xmit_one(struct virtnet_info *vi,  in __virtnet_xdp_xmit_one()  argument
464 	if (unlikely(xdpf->headroom < vi->hdr_len))  in __virtnet_xdp_xmit_one()
468 	xdpf->data -= vi->hdr_len;  in __virtnet_xdp_xmit_one()
471 	memset(hdr, 0, vi->hdr_len);  in __virtnet_xdp_xmit_one()
472 	xdpf->len   += vi->hdr_len;  in __virtnet_xdp_xmit_one()
484 static struct send_queue *virtnet_xdp_sq(struct virtnet_info *vi)  in virtnet_xdp_sq()  argument
488 	qp = vi->curr_queue_pairs - vi->xdp_queue_pairs + smp_processor_id();  in virtnet_xdp_sq()
489 	return &vi->sq[qp];  in virtnet_xdp_sq()
495 	struct virtnet_info *vi = netdev_priv(dev);  in virtnet_xdp_xmit()  local
496 	struct receive_queue *rq = vi->rq;  in virtnet_xdp_xmit()
515 	sq = virtnet_xdp_sq(vi);  in virtnet_xdp_xmit()
542 		err = __virtnet_xdp_xmit_one(vi, sq, xdpf);  in virtnet_xdp_xmit()
566 static unsigned int virtnet_get_headroom(struct virtnet_info *vi)  in virtnet_get_headroom()  argument
568 	return vi->xdp_queue_pairs ? VIRTIO_XDP_HEADROOM : 0;  in virtnet_get_headroom()
636 				     struct virtnet_info *vi,  in receive_small()  argument
647 	unsigned int headroom = vi->hdr_len + header_offset;  in receive_small()
656 	len -= vi->hdr_len;  in receive_small()
671 		if (unlikely(xdp_headroom < virtnet_get_headroom(vi))) {  in receive_small()
673 			unsigned int tlen = len + vi->hdr_len;  in receive_small()
676 			xdp_headroom = virtnet_get_headroom(vi);  in receive_small()
678 			headroom = vi->hdr_len + header_offset;  in receive_small()
692 		xdp.data_hard_start = buf + VIRTNET_RX_PAD + vi->hdr_len;  in receive_small()
716 				trace_xdp_exception(vi->dev, xdp_prog, act);  in receive_small()
734 			trace_xdp_exception(vi->dev, xdp_prog, act);  in receive_small()
750 		memcpy(skb_vnet_hdr(skb), buf, vi->hdr_len);  in receive_small()
769 				   struct virtnet_info *vi,  in receive_big()  argument
777 		page_to_skb(vi, rq, page, 0, len, PAGE_SIZE, true, 0);  in receive_big()
779 	stats->bytes += len - vi->hdr_len;  in receive_big()
792 					 struct virtnet_info *vi,  in receive_mergeable()  argument
801 	u16 num_buf = virtio16_to_cpu(vi->vdev, hdr->num_buffers);  in receive_mergeable()
813 	stats->bytes += len - vi->hdr_len;  in receive_mergeable()
843 			     headroom < virtnet_get_headroom(vi))) {  in receive_mergeable()
862 		xdp.data_hard_start = data - VIRTIO_XDP_HEADROOM + vi->hdr_len;  in receive_mergeable()
863 		xdp.data = data + vi->hdr_len;  in receive_mergeable()
864 		xdp.data_end = xdp.data + (len - vi->hdr_len);  in receive_mergeable()
867 		xdp.frame_sz = frame_sz - vi->hdr_len;  in receive_mergeable()
882 				 vi->hdr_len - metasize;  in receive_mergeable()
887 			len = xdp.data_end - xdp.data + vi->hdr_len + metasize;  in receive_mergeable()
892 				head_skb = page_to_skb(vi, rq, xdp_page, offset,  in receive_mergeable()
905 				trace_xdp_exception(vi->dev, xdp_prog, act);  in receive_mergeable()
932 			trace_xdp_exception(vi->dev, xdp_prog, act);  in receive_mergeable()
949 	head_skb = page_to_skb(vi, rq, page, offset, len, truesize, !xdp_prog,  in receive_mergeable()
962 				 virtio16_to_cpu(vi->vdev,  in receive_mergeable()
1036 static void receive_buf(struct virtnet_info *vi, struct receive_queue *rq,  in receive_buf()  argument
1041 	struct net_device *dev = vi->dev;  in receive_buf()
1045 	if (unlikely(len < vi->hdr_len + ETH_HLEN)) {  in receive_buf()
1048 		if (vi->mergeable_rx_bufs) {  in receive_buf()
1050 		} else if (vi->big_packets) {  in receive_buf()
1058 	if (vi->mergeable_rx_bufs)  in receive_buf()
1059 		skb = receive_mergeable(dev, vi, rq, buf, ctx, len, xdp_xmit,  in receive_buf()
1061 	else if (vi->big_packets)  in receive_buf()
1062 		skb = receive_big(dev, vi, rq, buf, len, stats);  in receive_buf()
1064 		skb = receive_small(dev, vi, rq, buf, ctx, len, xdp_xmit, stats);  in receive_buf()
1075 				  virtio_is_little_endian(vi->vdev))) {  in receive_buf()
1100 static int add_recvbuf_small(struct virtnet_info *vi, struct receive_queue *rq,  in add_recvbuf_small()  argument
1105 	unsigned int xdp_headroom = virtnet_get_headroom(vi);  in add_recvbuf_small()
1107 	int len = vi->hdr_len + VIRTNET_RX_PAD + GOOD_PACKET_LEN + xdp_headroom;  in add_recvbuf_small()
1119 		    vi->hdr_len + GOOD_PACKET_LEN);  in add_recvbuf_small()
1126 static int add_recvbuf_big(struct virtnet_info *vi, struct receive_queue *rq,  in add_recvbuf_big()  argument
1159 	sg_set_buf(&rq->sg[0], p, vi->hdr_len);  in add_recvbuf_big()
1191 static int add_recvbuf_mergeable(struct virtnet_info *vi,  in add_recvbuf_mergeable()  argument
1195 	unsigned int headroom = virtnet_get_headroom(vi);  in add_recvbuf_mergeable()
1241 static bool try_fill_recv(struct virtnet_info *vi, struct receive_queue *rq,  in try_fill_recv()  argument
1248 		if (vi->mergeable_rx_bufs)  in try_fill_recv()
1249 			err = add_recvbuf_mergeable(vi, rq, gfp);  in try_fill_recv()
1250 		else if (vi->big_packets)  in try_fill_recv()
1251 			err = add_recvbuf_big(vi, rq, gfp);  in try_fill_recv()
1253 			err = add_recvbuf_small(vi, rq, gfp);  in try_fill_recv()
1272 	struct virtnet_info *vi = rvq->vdev->priv;  in skb_recv_done()  local
1273 	struct receive_queue *rq = &vi->rq[vq2rxq(rvq)];  in skb_recv_done()
1291 static void virtnet_napi_tx_enable(struct virtnet_info *vi,  in virtnet_napi_tx_enable()  argument
1301 	if (!vi->affinity_hint_set) {  in virtnet_napi_tx_enable()
1317 	struct virtnet_info *vi =  in refill_work()  local
1322 	for (i = 0; i < vi->curr_queue_pairs; i++) {  in refill_work()
1323 		struct receive_queue *rq = &vi->rq[i];  in refill_work()
1326 		still_empty = !try_fill_recv(vi, rq, GFP_KERNEL);  in refill_work()
1333 			schedule_delayed_work(&vi->refill, HZ/2);  in refill_work()
1340 	struct virtnet_info *vi = rq->vq->vdev->priv;  in virtnet_receive()  local
1346 	if (!vi->big_packets || vi->mergeable_rx_bufs) {  in virtnet_receive()
1351 			receive_buf(vi, rq, buf, len, ctx, xdp_xmit, &stats);  in virtnet_receive()
1357 			receive_buf(vi, rq, buf, len, NULL, xdp_xmit, &stats);  in virtnet_receive()
1363 		if (!try_fill_recv(vi, rq, GFP_ATOMIC))  in virtnet_receive()
1364 			schedule_delayed_work(&vi->refill, 0);  in virtnet_receive()
1416 static bool is_xdp_raw_buffer_queue(struct virtnet_info *vi, int q)  in is_xdp_raw_buffer_queue()  argument
1418 	if (q < (vi->curr_queue_pairs - vi->xdp_queue_pairs))  in is_xdp_raw_buffer_queue()
1420 	else if (q < vi->curr_queue_pairs)  in is_xdp_raw_buffer_queue()
1428 	struct virtnet_info *vi = rq->vq->vdev->priv;  in virtnet_poll_cleantx()  local
1430 	struct send_queue *sq = &vi->sq[index];  in virtnet_poll_cleantx()
1431 	struct netdev_queue *txq = netdev_get_tx_queue(vi->dev, index);  in virtnet_poll_cleantx()
1433 	if (!sq->napi.weight || is_xdp_raw_buffer_queue(vi, index))  in virtnet_poll_cleantx()
1449 	struct virtnet_info *vi = rq->vq->vdev->priv;  in virtnet_poll()  local
1466 		sq = virtnet_xdp_sq(vi);  in virtnet_poll()
1479 	struct virtnet_info *vi = netdev_priv(dev);  in virtnet_open()  local
1482 	for (i = 0; i < vi->max_queue_pairs; i++) {  in virtnet_open()
1483 		if (i < vi->curr_queue_pairs)  in virtnet_open()
1485 			if (!try_fill_recv(vi, &vi->rq[i], GFP_KERNEL))  in virtnet_open()
1486 				schedule_delayed_work(&vi->refill, 0);  in virtnet_open()
1488 		err = xdp_rxq_info_reg(&vi->rq[i].xdp_rxq, dev, i);  in virtnet_open()
1492 		err = xdp_rxq_info_reg_mem_model(&vi->rq[i].xdp_rxq,  in virtnet_open()
1495 			xdp_rxq_info_unreg(&vi->rq[i].xdp_rxq);  in virtnet_open()
1499 		virtnet_napi_enable(vi->rq[i].vq, &vi->rq[i].napi);  in virtnet_open()
1500 		virtnet_napi_tx_enable(vi, vi->sq[i].vq, &vi->sq[i].napi);  in virtnet_open()
1509 	struct virtnet_info *vi = sq->vq->vdev->priv;  in virtnet_poll_tx()  local
1513 	if (unlikely(is_xdp_raw_buffer_queue(vi, index))) {  in virtnet_poll_tx()
1519 	txq = netdev_get_tx_queue(vi->dev, index);  in virtnet_poll_tx()
1536 	struct virtnet_info *vi = sq->vq->vdev->priv;  in xmit_skb()  local
1538 	unsigned hdr_len = vi->hdr_len;  in xmit_skb()
1541 	pr_debug("%s: xmit %p %pM\n", vi->dev->name, skb, dest);  in xmit_skb()
1543 	can_push = vi->any_header_sg &&  in xmit_skb()
1554 				    virtio_is_little_endian(vi->vdev), false,  in xmit_skb()
1558 	if (vi->mergeable_rx_bufs)  in xmit_skb()
1581 	struct virtnet_info *vi = netdev_priv(dev);  in start_xmit()  local
1583 	struct send_queue *sq = &vi->sq[qnum];  in start_xmit()
1658 static bool virtnet_send_command(struct virtnet_info *vi, u8 class, u8 cmd,  in virtnet_send_command()  argument
1665 	BUG_ON(!virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_VQ));  in virtnet_send_command()
1667 	vi->ctrl->status = ~0;  in virtnet_send_command()
1668 	vi->ctrl->hdr.class = class;  in virtnet_send_command()
1669 	vi->ctrl->hdr.cmd = cmd;  in virtnet_send_command()
1671 	sg_init_one(&hdr, &vi->ctrl->hdr, sizeof(vi->ctrl->hdr));  in virtnet_send_command()
1678 	sg_init_one(&stat, &vi->ctrl->status, sizeof(vi->ctrl->status));  in virtnet_send_command()
1682 	virtqueue_add_sgs(vi->cvq, sgs, out_num, 1, vi, GFP_ATOMIC);  in virtnet_send_command()
1684 	if (unlikely(!virtqueue_kick(vi->cvq)))  in virtnet_send_command()
1685 		return vi->ctrl->status == VIRTIO_NET_OK;  in virtnet_send_command()
1690 	while (!virtqueue_get_buf(vi->cvq, &tmp) &&  in virtnet_send_command()
1691 	       !virtqueue_is_broken(vi->cvq))  in virtnet_send_command()
1694 	return vi->ctrl->status == VIRTIO_NET_OK;  in virtnet_send_command()
1699 	struct virtnet_info *vi = netdev_priv(dev);  in virtnet_set_mac_address()  local
1700 	struct virtio_device *vdev = vi->vdev;  in virtnet_set_mac_address()
1705 	if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_STANDBY))  in virtnet_set_mac_address()
1718 		if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_MAC,  in virtnet_set_mac_address()
1747 	struct virtnet_info *vi = netdev_priv(dev);  in virtnet_stats()  local
1751 	for (i = 0; i < vi->max_queue_pairs; i++) {  in virtnet_stats()
1753 		struct receive_queue *rq = &vi->rq[i];  in virtnet_stats()
1754 		struct send_queue *sq = &vi->sq[i];  in virtnet_stats()
1782 static void virtnet_ack_link_announce(struct virtnet_info *vi)  in virtnet_ack_link_announce()  argument
1785 	if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_ANNOUNCE,  in virtnet_ack_link_announce()
1787 		dev_warn(&vi->dev->dev, "Failed to ack link announce.\n");  in virtnet_ack_link_announce()
1791 static int _virtnet_set_queues(struct virtnet_info *vi, u16 queue_pairs)  in _virtnet_set_queues()  argument
1794 	struct net_device *dev = vi->dev;  in _virtnet_set_queues()
1796 	if (!vi->has_cvq || !virtio_has_feature(vi->vdev, VIRTIO_NET_F_MQ))  in _virtnet_set_queues()
1799 	vi->ctrl->mq.virtqueue_pairs = cpu_to_virtio16(vi->vdev, queue_pairs);  in _virtnet_set_queues()
1800 	sg_init_one(&sg, &vi->ctrl->mq, sizeof(vi->ctrl->mq));  in _virtnet_set_queues()
1802 	if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_MQ,  in _virtnet_set_queues()
1808 		vi->curr_queue_pairs = queue_pairs;  in _virtnet_set_queues()
1811 			schedule_delayed_work(&vi->refill, 0);  in _virtnet_set_queues()
1817 static int virtnet_set_queues(struct virtnet_info *vi, u16 queue_pairs)  in virtnet_set_queues()  argument
1822 	err = _virtnet_set_queues(vi, queue_pairs);  in virtnet_set_queues()
1829 	struct virtnet_info *vi = netdev_priv(dev);  in virtnet_close()  local
1833 	cancel_delayed_work_sync(&vi->refill);  in virtnet_close()
1835 	for (i = 0; i < vi->max_queue_pairs; i++) {  in virtnet_close()
1836 		xdp_rxq_info_unreg(&vi->rq[i].xdp_rxq);  in virtnet_close()
1837 		napi_disable(&vi->rq[i].napi);  in virtnet_close()
1838 		virtnet_napi_tx_disable(&vi->sq[i].napi);  in virtnet_close()
1846 	struct virtnet_info *vi = netdev_priv(dev);  in virtnet_set_rx_mode()  local
1856 	if (!virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_RX))  in virtnet_set_rx_mode()
1859 	vi->ctrl->promisc = ((dev->flags & IFF_PROMISC) != 0);  in virtnet_set_rx_mode()
1860 	vi->ctrl->allmulti = ((dev->flags & IFF_ALLMULTI) != 0);  in virtnet_set_rx_mode()
1862 	sg_init_one(sg, &vi->ctrl->promisc, sizeof(vi->ctrl->promisc));  in virtnet_set_rx_mode()
1864 	if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_RX,  in virtnet_set_rx_mode()
1867 			 vi->ctrl->promisc ? "en" : "dis");  in virtnet_set_rx_mode()
1869 	sg_init_one(sg, &vi->ctrl->allmulti, sizeof(vi->ctrl->allmulti));  in virtnet_set_rx_mode()
1871 	if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_RX,  in virtnet_set_rx_mode()
1874 			 vi->ctrl->allmulti ? "en" : "dis");  in virtnet_set_rx_mode()
1888 	mac_data->entries = cpu_to_virtio32(vi->vdev, uc_count);  in virtnet_set_rx_mode()
1899 	mac_data->entries = cpu_to_virtio32(vi->vdev, mc_count);  in virtnet_set_rx_mode()
1907 	if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_MAC,  in virtnet_set_rx_mode()
1917 	struct virtnet_info *vi = netdev_priv(dev);  in virtnet_vlan_rx_add_vid()  local
1920 	vi->ctrl->vid = cpu_to_virtio16(vi->vdev, vid);  in virtnet_vlan_rx_add_vid()
1921 	sg_init_one(&sg, &vi->ctrl->vid, sizeof(vi->ctrl->vid));  in virtnet_vlan_rx_add_vid()
1923 	if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_VLAN,  in virtnet_vlan_rx_add_vid()
1932 	struct virtnet_info *vi = netdev_priv(dev);  in virtnet_vlan_rx_kill_vid()  local
1935 	vi->ctrl->vid = cpu_to_virtio16(vi->vdev, vid);  in virtnet_vlan_rx_kill_vid()
1936 	sg_init_one(&sg, &vi->ctrl->vid, sizeof(vi->ctrl->vid));  in virtnet_vlan_rx_kill_vid()
1938 	if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_VLAN,  in virtnet_vlan_rx_kill_vid()
1944 static void virtnet_clean_affinity(struct virtnet_info *vi)  in virtnet_clean_affinity()  argument
1948 	if (vi->affinity_hint_set) {  in virtnet_clean_affinity()
1949 		for (i = 0; i < vi->max_queue_pairs; i++) {  in virtnet_clean_affinity()
1950 			virtqueue_set_affinity(vi->rq[i].vq, NULL);  in virtnet_clean_affinity()
1951 			virtqueue_set_affinity(vi->sq[i].vq, NULL);  in virtnet_clean_affinity()
1954 		vi->affinity_hint_set = false;  in virtnet_clean_affinity()
1958 static void virtnet_set_affinity(struct virtnet_info *vi)  in virtnet_set_affinity()  argument
1968 		virtnet_clean_affinity(vi);  in virtnet_set_affinity()
1973 	stride = max_t(int, num_cpu / vi->curr_queue_pairs, 1);  in virtnet_set_affinity()
1974 	stragglers = num_cpu >= vi->curr_queue_pairs ?  in virtnet_set_affinity()
1975 			num_cpu % vi->curr_queue_pairs :  in virtnet_set_affinity()
1979 	for (i = 0; i < vi->curr_queue_pairs; i++) {  in virtnet_set_affinity()
1987 		virtqueue_set_affinity(vi->rq[i].vq, mask);  in virtnet_set_affinity()
1988 		virtqueue_set_affinity(vi->sq[i].vq, mask);  in virtnet_set_affinity()
1989 		__netif_set_xps_queue(vi->dev, cpumask_bits(mask), i, false);  in virtnet_set_affinity()
1993 	vi->affinity_hint_set = true;  in virtnet_set_affinity()
1999 	struct virtnet_info *vi = hlist_entry_safe(node, struct virtnet_info,  in virtnet_cpu_online()  local
2001 	virtnet_set_affinity(vi);  in virtnet_cpu_online()
2007 	struct virtnet_info *vi = hlist_entry_safe(node, struct virtnet_info,  in virtnet_cpu_dead()  local
2009 	virtnet_set_affinity(vi);  in virtnet_cpu_dead()
2015 	struct virtnet_info *vi = hlist_entry_safe(node, struct virtnet_info,  in virtnet_cpu_down_prep()  local
2018 	virtnet_clean_affinity(vi);  in virtnet_cpu_down_prep()
2024 static int virtnet_cpu_notif_add(struct virtnet_info *vi)  in virtnet_cpu_notif_add()  argument
2028 	ret = cpuhp_state_add_instance_nocalls(virtionet_online, &vi->node);  in virtnet_cpu_notif_add()
2032 					       &vi->node_dead);  in virtnet_cpu_notif_add()
2035 	cpuhp_state_remove_instance_nocalls(virtionet_online, &vi->node);  in virtnet_cpu_notif_add()
2039 static void virtnet_cpu_notif_remove(struct virtnet_info *vi)  in virtnet_cpu_notif_remove()  argument
2041 	cpuhp_state_remove_instance_nocalls(virtionet_online, &vi->node);  in virtnet_cpu_notif_remove()
2043 					    &vi->node_dead);  in virtnet_cpu_notif_remove()
2049 	struct virtnet_info *vi = netdev_priv(dev);  in virtnet_get_ringparam()  local
2051 	ring->rx_max_pending = virtqueue_get_vring_size(vi->rq[0].vq);  in virtnet_get_ringparam()
2052 	ring->tx_max_pending = virtqueue_get_vring_size(vi->sq[0].vq);  in virtnet_get_ringparam()
2061 	struct virtnet_info *vi = netdev_priv(dev);  in virtnet_get_drvinfo()  local
2062 	struct virtio_device *vdev = vi->vdev;  in virtnet_get_drvinfo()
2074 	struct virtnet_info *vi = netdev_priv(dev);  in virtnet_set_channels()  local
2084 	if (queue_pairs > vi->max_queue_pairs || queue_pairs == 0)  in virtnet_set_channels()
2091 	if (vi->rq[0].xdp_prog)  in virtnet_set_channels()
2095 	err = _virtnet_set_queues(vi, queue_pairs);  in virtnet_set_channels()
2100 		virtnet_set_affinity(vi);  in virtnet_set_channels()
2109 	struct virtnet_info *vi = netdev_priv(dev);  in virtnet_get_strings()  local
2115 		for (i = 0; i < vi->curr_queue_pairs; i++) {  in virtnet_get_strings()
2123 		for (i = 0; i < vi->curr_queue_pairs; i++) {  in virtnet_get_strings()
2136 	struct virtnet_info *vi = netdev_priv(dev);  in virtnet_get_sset_count()  local
2140 		return vi->curr_queue_pairs * (VIRTNET_RQ_STATS_LEN +  in virtnet_get_sset_count()
2150 	struct virtnet_info *vi = netdev_priv(dev);  in virtnet_get_ethtool_stats()  local
2155 	for (i = 0; i < vi->curr_queue_pairs; i++) {  in virtnet_get_ethtool_stats()
2156 		struct receive_queue *rq = &vi->rq[i];  in virtnet_get_ethtool_stats()
2169 	for (i = 0; i < vi->curr_queue_pairs; i++) {  in virtnet_get_ethtool_stats()
2170 		struct send_queue *sq = &vi->sq[i];  in virtnet_get_ethtool_stats()
2187 	struct virtnet_info *vi = netdev_priv(dev);  in virtnet_get_channels()  local
2189 	channels->combined_count = vi->curr_queue_pairs;  in virtnet_get_channels()
2190 	channels->max_combined = vi->max_queue_pairs;  in virtnet_get_channels()
2200 	struct virtnet_info *vi = netdev_priv(dev);  in virtnet_set_link_ksettings()  local
2203 						  &vi->speed, &vi->duplex);  in virtnet_set_link_ksettings()
2209 	struct virtnet_info *vi = netdev_priv(dev);  in virtnet_get_link_ksettings()  local
2211 	cmd->base.speed = vi->speed;  in virtnet_get_link_ksettings()
2212 	cmd->base.duplex = vi->duplex;  in virtnet_get_link_ksettings()
2221 	struct virtnet_info *vi = netdev_priv(dev);  in virtnet_set_coalesce()  local
2229 	if (napi_weight ^ vi->sq[0].napi.weight) {  in virtnet_set_coalesce()
2232 		for (i = 0; i < vi->max_queue_pairs; i++)  in virtnet_set_coalesce()
2233 			vi->sq[i].napi.weight = napi_weight;  in virtnet_set_coalesce()
2246 	struct virtnet_info *vi = netdev_priv(dev);  in virtnet_get_coalesce()  local
2250 	if (vi->sq[0].napi.weight)  in virtnet_get_coalesce()
2258 	struct virtnet_info *vi = netdev_priv(dev);  in virtnet_init_settings()  local
2260 	vi->speed = SPEED_UNKNOWN;  in virtnet_init_settings()
2261 	vi->duplex = DUPLEX_UNKNOWN;  in virtnet_init_settings()
2264 static void virtnet_update_settings(struct virtnet_info *vi)  in virtnet_update_settings()  argument
2269 	if (!virtio_has_feature(vi->vdev, VIRTIO_NET_F_SPEED_DUPLEX))  in virtnet_update_settings()
2272 	virtio_cread_le(vi->vdev, struct virtio_net_config, speed, &speed);  in virtnet_update_settings()
2275 		vi->speed = speed;  in virtnet_update_settings()
2277 	virtio_cread_le(vi->vdev, struct virtio_net_config, duplex, &duplex);  in virtnet_update_settings()
2280 		vi->duplex = duplex;  in virtnet_update_settings()
2302 	struct virtnet_info *vi = vdev->priv;  in virtnet_freeze_down()  local
2306 	flush_work(&vi->config_work);  in virtnet_freeze_down()
2308 	netif_tx_lock_bh(vi->dev);  in virtnet_freeze_down()
2309 	netif_device_detach(vi->dev);  in virtnet_freeze_down()
2310 	netif_tx_unlock_bh(vi->dev);  in virtnet_freeze_down()
2311 	cancel_delayed_work_sync(&vi->refill);  in virtnet_freeze_down()
2313 	if (netif_running(vi->dev)) {  in virtnet_freeze_down()
2314 		for (i = 0; i < vi->max_queue_pairs; i++) {  in virtnet_freeze_down()
2315 			napi_disable(&vi->rq[i].napi);  in virtnet_freeze_down()
2316 			virtnet_napi_tx_disable(&vi->sq[i].napi);  in virtnet_freeze_down()
2321 static int init_vqs(struct virtnet_info *vi);
2325 	struct virtnet_info *vi = vdev->priv;  in virtnet_restore_up()  local
2328 	err = init_vqs(vi);  in virtnet_restore_up()
2334 	if (netif_running(vi->dev)) {  in virtnet_restore_up()
2335 		for (i = 0; i < vi->curr_queue_pairs; i++)  in virtnet_restore_up()
2336 			if (!try_fill_recv(vi, &vi->rq[i], GFP_KERNEL))  in virtnet_restore_up()
2337 				schedule_delayed_work(&vi->refill, 0);  in virtnet_restore_up()
2339 		for (i = 0; i < vi->max_queue_pairs; i++) {  in virtnet_restore_up()
2340 			virtnet_napi_enable(vi->rq[i].vq, &vi->rq[i].napi);  in virtnet_restore_up()
2341 			virtnet_napi_tx_enable(vi, vi->sq[i].vq,  in virtnet_restore_up()
2342 					       &vi->sq[i].napi);  in virtnet_restore_up()
2346 	netif_tx_lock_bh(vi->dev);  in virtnet_restore_up()
2347 	netif_device_attach(vi->dev);  in virtnet_restore_up()
2348 	netif_tx_unlock_bh(vi->dev);  in virtnet_restore_up()
2352 static int virtnet_set_guest_offloads(struct virtnet_info *vi, u64 offloads)  in virtnet_set_guest_offloads()  argument
2355 	vi->ctrl->offloads = cpu_to_virtio64(vi->vdev, offloads);  in virtnet_set_guest_offloads()
2357 	sg_init_one(&sg, &vi->ctrl->offloads, sizeof(vi->ctrl->offloads));  in virtnet_set_guest_offloads()
2359 	if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_GUEST_OFFLOADS,  in virtnet_set_guest_offloads()
2361 		dev_warn(&vi->dev->dev, "Fail to set guest offload.\n");  in virtnet_set_guest_offloads()
2368 static int virtnet_clear_guest_offloads(struct virtnet_info *vi)  in virtnet_clear_guest_offloads()  argument
2372 	if (!vi->guest_offloads)  in virtnet_clear_guest_offloads()
2375 	return virtnet_set_guest_offloads(vi, offloads);  in virtnet_clear_guest_offloads()
2378 static int virtnet_restore_guest_offloads(struct virtnet_info *vi)  in virtnet_restore_guest_offloads()  argument
2380 	u64 offloads = vi->guest_offloads;  in virtnet_restore_guest_offloads()
2382 	if (!vi->guest_offloads)  in virtnet_restore_guest_offloads()
2385 	return virtnet_set_guest_offloads(vi, offloads);  in virtnet_restore_guest_offloads()
2392 	struct virtnet_info *vi = netdev_priv(dev);  in virtnet_xdp_set()  local
2397 	if (!virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_GUEST_OFFLOADS)  in virtnet_xdp_set()
2398 	    && (virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_TSO4) ||  in virtnet_xdp_set()
2399 	        virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_TSO6) ||  in virtnet_xdp_set()
2400 	        virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_ECN) ||  in virtnet_xdp_set()
2401 		virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_UFO) ||  in virtnet_xdp_set()
2402 		virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_CSUM))) {  in virtnet_xdp_set()
2407 	if (vi->mergeable_rx_bufs && !vi->any_header_sg) {  in virtnet_xdp_set()
2418 	curr_qp = vi->curr_queue_pairs - vi->xdp_queue_pairs;  in virtnet_xdp_set()
2423 	if (curr_qp + xdp_qp > vi->max_queue_pairs) {  in virtnet_xdp_set()
2426 			    curr_qp + xdp_qp, vi->max_queue_pairs);  in virtnet_xdp_set()
2430 	old_prog = rtnl_dereference(vi->rq[0].xdp_prog);  in virtnet_xdp_set()
2435 		bpf_prog_add(prog, vi->max_queue_pairs - 1);  in virtnet_xdp_set()
2439 		for (i = 0; i < vi->max_queue_pairs; i++) {  in virtnet_xdp_set()
2440 			napi_disable(&vi->rq[i].napi);  in virtnet_xdp_set()
2441 			virtnet_napi_tx_disable(&vi->sq[i].napi);  in virtnet_xdp_set()
2446 		for (i = 0; i < vi->max_queue_pairs; i++) {  in virtnet_xdp_set()
2447 			rcu_assign_pointer(vi->rq[i].xdp_prog, prog);  in virtnet_xdp_set()
2449 				virtnet_restore_guest_offloads(vi);  in virtnet_xdp_set()
2454 	err = _virtnet_set_queues(vi, curr_qp + xdp_qp);  in virtnet_xdp_set()
2458 	vi->xdp_queue_pairs = xdp_qp;  in virtnet_xdp_set()
2461 		for (i = 0; i < vi->max_queue_pairs; i++) {  in virtnet_xdp_set()
2462 			rcu_assign_pointer(vi->rq[i].xdp_prog, prog);  in virtnet_xdp_set()
2464 				virtnet_clear_guest_offloads(vi);  in virtnet_xdp_set()
2468 	for (i = 0; i < vi->max_queue_pairs; i++) {  in virtnet_xdp_set()
2472 			virtnet_napi_enable(vi->rq[i].vq, &vi->rq[i].napi);  in virtnet_xdp_set()
2473 			virtnet_napi_tx_enable(vi, vi->sq[i].vq,  in virtnet_xdp_set()
2474 					       &vi->sq[i].napi);  in virtnet_xdp_set()
2482 		virtnet_clear_guest_offloads(vi);  in virtnet_xdp_set()
2483 		for (i = 0; i < vi->max_queue_pairs; i++)  in virtnet_xdp_set()
2484 			rcu_assign_pointer(vi->rq[i].xdp_prog, old_prog);  in virtnet_xdp_set()
2488 		for (i = 0; i < vi->max_queue_pairs; i++) {  in virtnet_xdp_set()
2489 			virtnet_napi_enable(vi->rq[i].vq, &vi->rq[i].napi);  in virtnet_xdp_set()
2490 			virtnet_napi_tx_enable(vi, vi->sq[i].vq,  in virtnet_xdp_set()
2491 					       &vi->sq[i].napi);  in virtnet_xdp_set()
2495 		bpf_prog_sub(prog, vi->max_queue_pairs - 1);  in virtnet_xdp_set()
2512 	struct virtnet_info *vi = netdev_priv(dev);  in virtnet_get_phys_port_name()  local
2515 	if (!virtio_has_feature(vi->vdev, VIRTIO_NET_F_STANDBY))  in virtnet_get_phys_port_name()
2528 	struct virtnet_info *vi = netdev_priv(dev);  in virtnet_set_features()  local
2533 		if (vi->xdp_queue_pairs)  in virtnet_set_features()
2537 			offloads = vi->guest_offloads_capable;  in virtnet_set_features()
2539 			offloads = vi->guest_offloads_capable &  in virtnet_set_features()
2542 		err = virtnet_set_guest_offloads(vi, offloads);  in virtnet_set_features()
2545 		vi->guest_offloads = offloads;  in virtnet_set_features()
2570 	struct virtnet_info *vi =  in virtnet_config_changed_work()  local
2574 	if (virtio_cread_feature(vi->vdev, VIRTIO_NET_F_STATUS,  in virtnet_config_changed_work()
2579 		netdev_notify_peers(vi->dev);  in virtnet_config_changed_work()
2580 		virtnet_ack_link_announce(vi);  in virtnet_config_changed_work()
2586 	if (vi->status == v)  in virtnet_config_changed_work()
2589 	vi->status = v;  in virtnet_config_changed_work()
2591 	if (vi->status & VIRTIO_NET_S_LINK_UP) {  in virtnet_config_changed_work()
2592 		virtnet_update_settings(vi);  in virtnet_config_changed_work()
2593 		netif_carrier_on(vi->dev);  in virtnet_config_changed_work()
2594 		netif_tx_wake_all_queues(vi->dev);  in virtnet_config_changed_work()
2596 		netif_carrier_off(vi->dev);  in virtnet_config_changed_work()
2597 		netif_tx_stop_all_queues(vi->dev);  in virtnet_config_changed_work()
2603 	struct virtnet_info *vi = vdev->priv;  in virtnet_config_changed()  local
2605 	schedule_work(&vi->config_work);  in virtnet_config_changed()
2608 static void virtnet_free_queues(struct virtnet_info *vi)  in virtnet_free_queues()  argument
2612 	for (i = 0; i < vi->max_queue_pairs; i++) {  in virtnet_free_queues()
2613 		__netif_napi_del(&vi->rq[i].napi);  in virtnet_free_queues()
2614 		__netif_napi_del(&vi->sq[i].napi);  in virtnet_free_queues()
2622 	kfree(vi->rq);  in virtnet_free_queues()
2623 	kfree(vi->sq);  in virtnet_free_queues()
2624 	kfree(vi->ctrl);  in virtnet_free_queues()
2627 static void _free_receive_bufs(struct virtnet_info *vi)  in _free_receive_bufs()  argument
2632 	for (i = 0; i < vi->max_queue_pairs; i++) {  in _free_receive_bufs()
2633 		while (vi->rq[i].pages)  in _free_receive_bufs()
2634 			__free_pages(get_a_page(&vi->rq[i], GFP_KERNEL), 0);  in _free_receive_bufs()
2636 		old_prog = rtnl_dereference(vi->rq[i].xdp_prog);  in _free_receive_bufs()
2637 		RCU_INIT_POINTER(vi->rq[i].xdp_prog, NULL);  in _free_receive_bufs()
2643 static void free_receive_bufs(struct virtnet_info *vi)  in free_receive_bufs()  argument
2646 	_free_receive_bufs(vi);  in free_receive_bufs()
2650 static void free_receive_page_frags(struct virtnet_info *vi)  in free_receive_page_frags()  argument
2653 	for (i = 0; i < vi->max_queue_pairs; i++)  in free_receive_page_frags()
2654 		if (vi->rq[i].alloc_frag.page)  in free_receive_page_frags()
2655 			put_page(vi->rq[i].alloc_frag.page);  in free_receive_page_frags()
2658 static void free_unused_bufs(struct virtnet_info *vi)  in free_unused_bufs()  argument
2663 	for (i = 0; i < vi->max_queue_pairs; i++) {  in free_unused_bufs()
2664 		struct virtqueue *vq = vi->sq[i].vq;  in free_unused_bufs()
2673 	for (i = 0; i < vi->max_queue_pairs; i++) {  in free_unused_bufs()
2674 		struct virtqueue *vq = vi->rq[i].vq;  in free_unused_bufs()
2677 			if (vi->mergeable_rx_bufs) {  in free_unused_bufs()
2679 			} else if (vi->big_packets) {  in free_unused_bufs()
2680 				give_pages(&vi->rq[i], buf);  in free_unused_bufs()
2688 static void virtnet_del_vqs(struct virtnet_info *vi)  in virtnet_del_vqs()  argument
2690 	struct virtio_device *vdev = vi->vdev;  in virtnet_del_vqs()
2692 	virtnet_clean_affinity(vi);  in virtnet_del_vqs()
2696 	virtnet_free_queues(vi);  in virtnet_del_vqs()
2703 static unsigned int mergeable_min_buf_len(struct virtnet_info *vi, struct virtqueue *vq)  in mergeable_min_buf_len()  argument
2707 	unsigned int packet_len = vi->big_packets ? IP_MAX_MTU : vi->dev->max_mtu;  in mergeable_min_buf_len()
2715 static int virtnet_find_vqs(struct virtnet_info *vi)  in virtnet_find_vqs()  argument
2728 	total_vqs = vi->max_queue_pairs * 2 +  in virtnet_find_vqs()
2729 		    virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_VQ);  in virtnet_find_vqs()
2741 	if (!vi->big_packets || vi->mergeable_rx_bufs) {  in virtnet_find_vqs()
2750 	if (vi->has_cvq) {  in virtnet_find_vqs()
2756 	for (i = 0; i < vi->max_queue_pairs; i++) {  in virtnet_find_vqs()
2759 		sprintf(vi->rq[i].name, "input.%d", i);  in virtnet_find_vqs()
2760 		sprintf(vi->sq[i].name, "output.%d", i);  in virtnet_find_vqs()
2761 		names[rxq2vq(i)] = vi->rq[i].name;  in virtnet_find_vqs()
2762 		names[txq2vq(i)] = vi->sq[i].name;  in virtnet_find_vqs()
2767 	ret = vi->vdev->config->find_vqs(vi->vdev, total_vqs, vqs, callbacks,  in virtnet_find_vqs()
2772 	if (vi->has_cvq) {  in virtnet_find_vqs()
2773 		vi->cvq = vqs[total_vqs - 1];  in virtnet_find_vqs()
2774 		if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_VLAN))  in virtnet_find_vqs()
2775 			vi->dev->features |= NETIF_F_HW_VLAN_CTAG_FILTER;  in virtnet_find_vqs()
2778 	for (i = 0; i < vi->max_queue_pairs; i++) {  in virtnet_find_vqs()
2779 		vi->rq[i].vq = vqs[rxq2vq(i)];  in virtnet_find_vqs()
2780 		vi->rq[i].min_buf_len = mergeable_min_buf_len(vi, vi->rq[i].vq);  in virtnet_find_vqs()
2781 		vi->sq[i].vq = vqs[txq2vq(i)];  in virtnet_find_vqs()
2799 static int virtnet_alloc_queues(struct virtnet_info *vi)  in virtnet_alloc_queues()  argument
2803 	vi->ctrl = kzalloc(sizeof(*vi->ctrl), GFP_KERNEL);  in virtnet_alloc_queues()
2804 	if (!vi->ctrl)  in virtnet_alloc_queues()
2806 	vi->sq = kcalloc(vi->max_queue_pairs, sizeof(*vi->sq), GFP_KERNEL);  in virtnet_alloc_queues()
2807 	if (!vi->sq)  in virtnet_alloc_queues()
2809 	vi->rq = kcalloc(vi->max_queue_pairs, sizeof(*vi->rq), GFP_KERNEL);  in virtnet_alloc_queues()
2810 	if (!vi->rq)  in virtnet_alloc_queues()
2813 	INIT_DELAYED_WORK(&vi->refill, refill_work);  in virtnet_alloc_queues()
2814 	for (i = 0; i < vi->max_queue_pairs; i++) {  in virtnet_alloc_queues()
2815 		vi->rq[i].pages = NULL;  in virtnet_alloc_queues()
2816 		netif_napi_add(vi->dev, &vi->rq[i].napi, virtnet_poll,  in virtnet_alloc_queues()
2818 		netif_tx_napi_add(vi->dev, &vi->sq[i].napi, virtnet_poll_tx,  in virtnet_alloc_queues()
2821 		sg_init_table(vi->rq[i].sg, ARRAY_SIZE(vi->rq[i].sg));  in virtnet_alloc_queues()
2822 		ewma_pkt_len_init(&vi->rq[i].mrg_avg_pkt_len);  in virtnet_alloc_queues()
2823 		sg_init_table(vi->sq[i].sg, ARRAY_SIZE(vi->sq[i].sg));  in virtnet_alloc_queues()
2825 		u64_stats_init(&vi->rq[i].stats.syncp);  in virtnet_alloc_queues()
2826 		u64_stats_init(&vi->sq[i].stats.syncp);  in virtnet_alloc_queues()
2832 	kfree(vi->sq);  in virtnet_alloc_queues()
2834 	kfree(vi->ctrl);  in virtnet_alloc_queues()
2839 static int init_vqs(struct virtnet_info *vi)  in init_vqs()  argument
2844 	ret = virtnet_alloc_queues(vi);  in init_vqs()
2848 	ret = virtnet_find_vqs(vi);  in init_vqs()
2853 	virtnet_set_affinity(vi);  in init_vqs()
2859 	virtnet_free_queues(vi);  in init_vqs()
2868 	struct virtnet_info *vi = netdev_priv(queue->dev);  in mergeable_rx_buffer_size_show()  local
2870 	unsigned int headroom = virtnet_get_headroom(vi);  in mergeable_rx_buffer_size_show()
2874 	BUG_ON(queue_index >= vi->max_queue_pairs);  in mergeable_rx_buffer_size_show()
2875 	avg = &vi->rq[queue_index].mrg_avg_pkt_len;  in mergeable_rx_buffer_size_show()
2877 		       get_mergeable_buf_len(&vi->rq[queue_index], avg,  in mergeable_rx_buffer_size_show()
2958 	struct virtnet_info *vi;  in virtnet_probe()  local
3034 	vi = netdev_priv(dev);  in virtnet_probe()
3035 	vi->dev = dev;  in virtnet_probe()
3036 	vi->vdev = vdev;  in virtnet_probe()
3037 	vdev->priv = vi;  in virtnet_probe()
3039 	INIT_WORK(&vi->config_work, virtnet_config_changed_work);  in virtnet_probe()
3046 		vi->big_packets = true;  in virtnet_probe()
3049 		vi->mergeable_rx_bufs = true;  in virtnet_probe()
3053 		vi->hdr_len = sizeof(struct virtio_net_hdr_mrg_rxbuf);  in virtnet_probe()
3055 		vi->hdr_len = sizeof(struct virtio_net_hdr);  in virtnet_probe()
3059 		vi->any_header_sg = true;  in virtnet_probe()
3062 		vi->has_cvq = true;  in virtnet_probe()
3083 			vi->big_packets = true;  in virtnet_probe()
3086 	if (vi->any_header_sg)  in virtnet_probe()
3087 		dev->needed_headroom = vi->hdr_len;  in virtnet_probe()
3091 		vi->curr_queue_pairs = max_queue_pairs;  in virtnet_probe()
3093 		vi->curr_queue_pairs = num_online_cpus();  in virtnet_probe()
3094 	vi->max_queue_pairs = max_queue_pairs;  in virtnet_probe()
3097 	err = init_vqs(vi);  in virtnet_probe()
3102 	if (vi->mergeable_rx_bufs)  in virtnet_probe()
3105 	netif_set_real_num_tx_queues(dev, vi->curr_queue_pairs);  in virtnet_probe()
3106 	netif_set_real_num_rx_queues(dev, vi->curr_queue_pairs);  in virtnet_probe()
3111 		vi->failover = net_failover_create(vi->dev);  in virtnet_probe()
3112 		if (IS_ERR(vi->failover)) {  in virtnet_probe()
3113 			err = PTR_ERR(vi->failover);  in virtnet_probe()
3126 	err = virtnet_cpu_notif_add(vi);  in virtnet_probe()
3132 	virtnet_set_queues(vi, vi->curr_queue_pairs);  in virtnet_probe()
3137 	if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_STATUS)) {  in virtnet_probe()
3138 		schedule_work(&vi->config_work);  in virtnet_probe()
3140 		vi->status = VIRTIO_NET_S_LINK_UP;  in virtnet_probe()
3141 		virtnet_update_settings(vi);  in virtnet_probe()
3146 		if (virtio_has_feature(vi->vdev, guest_offloads[i]))  in virtnet_probe()
3147 			set_bit(guest_offloads[i], &vi->guest_offloads);  in virtnet_probe()
3148 	vi->guest_offloads_capable = vi->guest_offloads;  in virtnet_probe()
3156 	vi->vdev->config->reset(vdev);  in virtnet_probe()
3160 	net_failover_destroy(vi->failover);  in virtnet_probe()
3162 	cancel_delayed_work_sync(&vi->refill);  in virtnet_probe()
3163 	free_receive_page_frags(vi);  in virtnet_probe()
3164 	virtnet_del_vqs(vi);  in virtnet_probe()
3170 static void remove_vq_common(struct virtnet_info *vi)  in remove_vq_common()  argument
3172 	vi->vdev->config->reset(vi->vdev);  in remove_vq_common()
3175 	free_unused_bufs(vi);  in remove_vq_common()
3177 	free_receive_bufs(vi);  in remove_vq_common()
3179 	free_receive_page_frags(vi);  in remove_vq_common()
3181 	virtnet_del_vqs(vi);  in remove_vq_common()
3186 	struct virtnet_info *vi = vdev->priv;  in virtnet_remove()  local
3188 	virtnet_cpu_notif_remove(vi);  in virtnet_remove()
3191 	flush_work(&vi->config_work);  in virtnet_remove()
3193 	unregister_netdev(vi->dev);  in virtnet_remove()
3195 	net_failover_destroy(vi->failover);  in virtnet_remove()
3197 	remove_vq_common(vi);  in virtnet_remove()
3199 	free_netdev(vi->dev);  in virtnet_remove()
3204 	struct virtnet_info *vi = vdev->priv;  in virtnet_freeze()  local
3206 	virtnet_cpu_notif_remove(vi);  in virtnet_freeze()
3208 	remove_vq_common(vi);  in virtnet_freeze()
3215 	struct virtnet_info *vi = vdev->priv;  in virtnet_restore()  local
3221 	virtnet_set_queues(vi, vi->curr_queue_pairs);  in virtnet_restore()
3223 	err = virtnet_cpu_notif_add(vi);  in virtnet_restore()