333 	struct virtnet_info *vi = vq->vdev->priv;  in skb_xmit_done()  local
334 	struct napi_struct *napi = &vi->sq[vq2txq(vq)].napi;  in skb_xmit_done()
343 		netif_wake_subqueue(vi->dev, vq2txq(vq));  in skb_xmit_done()
364 static struct sk_buff *page_to_skb(struct virtnet_info *vi,  in page_to_skb()  argument
383 	hdr_len = vi->hdr_len;  in page_to_skb()
384 	if (vi->mergeable_rx_bufs)  in page_to_skb()
403 	if (vi->mergeable_rx_bufs) {  in page_to_skb()
438 static int __virtnet_xdp_xmit_one(struct virtnet_info *vi,  in __virtnet_xdp_xmit_one()  argument
449 	if (unlikely(xdpf->headroom < vi->hdr_len))  in __virtnet_xdp_xmit_one()
453 	xdpf->data -= vi->hdr_len;  in __virtnet_xdp_xmit_one()
456 	memset(hdr, 0, vi->hdr_len);  in __virtnet_xdp_xmit_one()
457 	xdpf->len   += vi->hdr_len;  in __virtnet_xdp_xmit_one()
468 static struct send_queue *virtnet_xdp_sq(struct virtnet_info *vi)  in virtnet_xdp_sq()  argument
472 	qp = vi->curr_queue_pairs - vi->xdp_queue_pairs + smp_processor_id();  in virtnet_xdp_sq()
473 	return &vi->sq[qp];  in virtnet_xdp_sq()
479 	struct virtnet_info *vi = netdev_priv(dev);  in virtnet_xdp_xmit()  local
480 	struct receive_queue *rq = vi->rq;  in virtnet_xdp_xmit()
490 	sq = virtnet_xdp_sq(vi);  in virtnet_xdp_xmit()
515 		err = __virtnet_xdp_xmit_one(vi, sq, xdpf);  in virtnet_xdp_xmit()
537 static unsigned int virtnet_get_headroom(struct virtnet_info *vi)  in virtnet_get_headroom()  argument
539 	return vi->xdp_queue_pairs ? VIRTIO_XDP_HEADROOM : 0;  in virtnet_get_headroom()
607 				     struct virtnet_info *vi,  in receive_small()  argument
618 	unsigned int headroom = vi->hdr_len + header_offset;  in receive_small()
626 	len -= vi->hdr_len;  in receive_small()
641 		if (unlikely(xdp_headroom < virtnet_get_headroom(vi))) {  in receive_small()
643 			unsigned int tlen = len + vi->hdr_len;  in receive_small()
646 			xdp_headroom = virtnet_get_headroom(vi);  in receive_small()
648 			headroom = vi->hdr_len + header_offset;  in receive_small()
662 		xdp.data_hard_start = buf + VIRTNET_RX_PAD + vi->hdr_len;  in receive_small()
684 				trace_xdp_exception(vi->dev, xdp_prog, act);  in receive_small()
702 			trace_xdp_exception(vi->dev, xdp_prog, act);  in receive_small()
718 		memcpy(skb_vnet_hdr(skb), buf, vi->hdr_len);  in receive_small()
734 				   struct virtnet_info *vi,  in receive_big()  argument
741 	struct sk_buff *skb = page_to_skb(vi, rq, page, 0, len, PAGE_SIZE);  in receive_big()
743 	stats->bytes += len - vi->hdr_len;  in receive_big()
756 					 struct virtnet_info *vi,  in receive_mergeable()  argument
765 	u16 num_buf = virtio16_to_cpu(vi->vdev, hdr->num_buffers);  in receive_mergeable()
775 	stats->bytes += len - vi->hdr_len;  in receive_mergeable()
800 			     headroom < virtnet_get_headroom(vi))) {  in receive_mergeable()
817 		xdp.data_hard_start = data - VIRTIO_XDP_HEADROOM + vi->hdr_len;  in receive_mergeable()
818 		xdp.data = data + vi->hdr_len;  in receive_mergeable()
820 		xdp.data_end = xdp.data + (len - vi->hdr_len);  in receive_mergeable()
833 					page_address(xdp_page) - vi->hdr_len;  in receive_mergeable()
838 			len = xdp.data_end - xdp.data + vi->hdr_len;  in receive_mergeable()
843 				head_skb = page_to_skb(vi, rq, xdp_page,  in receive_mergeable()
855 				trace_xdp_exception(vi->dev, xdp_prog, act);  in receive_mergeable()
882 			trace_xdp_exception(vi->dev, xdp_prog, act);  in receive_mergeable()
900 	head_skb = page_to_skb(vi, rq, page, offset, len, truesize);  in receive_mergeable()
912 				 virtio16_to_cpu(vi->vdev,  in receive_mergeable()
986 static void receive_buf(struct virtnet_info *vi, struct receive_queue *rq,  in receive_buf()  argument
991 	struct net_device *dev = vi->dev;  in receive_buf()
995 	if (unlikely(len < vi->hdr_len + ETH_HLEN)) {  in receive_buf()
998 		if (vi->mergeable_rx_bufs) {  in receive_buf()
1000 		} else if (vi->big_packets) {  in receive_buf()
1008 	if (vi->mergeable_rx_bufs)  in receive_buf()
1009 		skb = receive_mergeable(dev, vi, rq, buf, ctx, len, xdp_xmit,  in receive_buf()
1011 	else if (vi->big_packets)  in receive_buf()
1012 		skb = receive_big(dev, vi, rq, buf, len, stats);  in receive_buf()
1014 		skb = receive_small(dev, vi, rq, buf, ctx, len, xdp_xmit, stats);  in receive_buf()
1025 				  virtio_is_little_endian(vi->vdev))) {  in receive_buf()
1049 static int add_recvbuf_small(struct virtnet_info *vi, struct receive_queue *rq,  in add_recvbuf_small()  argument
1054 	unsigned int xdp_headroom = virtnet_get_headroom(vi);  in add_recvbuf_small()
1056 	int len = vi->hdr_len + VIRTNET_RX_PAD + GOOD_PACKET_LEN + xdp_headroom;  in add_recvbuf_small()
1068 		    vi->hdr_len + GOOD_PACKET_LEN);  in add_recvbuf_small()
1075 static int add_recvbuf_big(struct virtnet_info *vi, struct receive_queue *rq,  in add_recvbuf_big()  argument
1108 	sg_set_buf(&rq->sg[0], p, vi->hdr_len);  in add_recvbuf_big()
1140 static int add_recvbuf_mergeable(struct virtnet_info *vi,  in add_recvbuf_mergeable()  argument
1144 	unsigned int headroom = virtnet_get_headroom(vi);  in add_recvbuf_mergeable()
1190 static bool try_fill_recv(struct virtnet_info *vi, struct receive_queue *rq,  in try_fill_recv()  argument
1197 		if (vi->mergeable_rx_bufs)  in try_fill_recv()
1198 			err = add_recvbuf_mergeable(vi, rq, gfp);  in try_fill_recv()
1199 		else if (vi->big_packets)  in try_fill_recv()
1200 			err = add_recvbuf_big(vi, rq, gfp);  in try_fill_recv()
1202 			err = add_recvbuf_small(vi, rq, gfp);  in try_fill_recv()
1219 	struct virtnet_info *vi = rvq->vdev->priv;  in skb_recv_done()  local
1220 	struct receive_queue *rq = &vi->rq[vq2rxq(rvq)];  in skb_recv_done()
1238 static void virtnet_napi_tx_enable(struct virtnet_info *vi,  in virtnet_napi_tx_enable()  argument
1248 	if (!vi->affinity_hint_set) {  in virtnet_napi_tx_enable()
1264 	struct virtnet_info *vi =  in refill_work()  local
1269 	for (i = 0; i < vi->curr_queue_pairs; i++) {  in refill_work()
1270 		struct receive_queue *rq = &vi->rq[i];  in refill_work()
1273 		still_empty = !try_fill_recv(vi, rq, GFP_KERNEL);  in refill_work()
1280 			schedule_delayed_work(&vi->refill, HZ/2);  in refill_work()
1287 	struct virtnet_info *vi = rq->vq->vdev->priv;  in virtnet_receive()  local
1293 	if (!vi->big_packets || vi->mergeable_rx_bufs) {  in virtnet_receive()
1298 			receive_buf(vi, rq, buf, len, ctx, xdp_xmit, &stats);  in virtnet_receive()
1304 			receive_buf(vi, rq, buf, len, NULL, xdp_xmit, &stats);  in virtnet_receive()
1310 		if (!try_fill_recv(vi, rq, GFP_ATOMIC))  in virtnet_receive()
1311 			schedule_delayed_work(&vi->refill, 0);  in virtnet_receive()
1357 	struct virtnet_info *vi = rq->vq->vdev->priv;  in virtnet_poll_cleantx()  local
1359 	struct send_queue *sq = &vi->sq[index];  in virtnet_poll_cleantx()
1360 	struct netdev_queue *txq = netdev_get_tx_queue(vi->dev, index);  in virtnet_poll_cleantx()
1378 	struct virtnet_info *vi = rq->vq->vdev->priv;  in virtnet_poll()  local
1395 		sq = virtnet_xdp_sq(vi);  in virtnet_poll()
1408 	struct virtnet_info *vi = netdev_priv(dev);  in virtnet_open()  local
1411 	for (i = 0; i < vi->max_queue_pairs; i++) {  in virtnet_open()
1412 		if (i < vi->curr_queue_pairs)  in virtnet_open()
1414 			if (!try_fill_recv(vi, &vi->rq[i], GFP_KERNEL))  in virtnet_open()
1415 				schedule_delayed_work(&vi->refill, 0);  in virtnet_open()
1417 		err = xdp_rxq_info_reg(&vi->rq[i].xdp_rxq, dev, i);  in virtnet_open()
1421 		err = xdp_rxq_info_reg_mem_model(&vi->rq[i].xdp_rxq,  in virtnet_open()
1424 			xdp_rxq_info_unreg(&vi->rq[i].xdp_rxq);  in virtnet_open()
1428 		virtnet_napi_enable(vi->rq[i].vq, &vi->rq[i].napi);  in virtnet_open()
1429 		virtnet_napi_tx_enable(vi, vi->sq[i].vq, &vi->sq[i].napi);  in virtnet_open()
1438 	struct virtnet_info *vi = sq->vq->vdev->priv;  in virtnet_poll_tx()  local
1439 	struct netdev_queue *txq = netdev_get_tx_queue(vi->dev, vq2txq(sq->vq));  in virtnet_poll_tx()
1457 	struct virtnet_info *vi = sq->vq->vdev->priv;  in xmit_skb()  local
1459 	unsigned hdr_len = vi->hdr_len;  in xmit_skb()
1462 	pr_debug("%s: xmit %p %pM\n", vi->dev->name, skb, dest);  in xmit_skb()
1464 	can_push = vi->any_header_sg &&  in xmit_skb()
1475 				    virtio_is_little_endian(vi->vdev), false,  in xmit_skb()
1479 	if (vi->mergeable_rx_bufs)  in xmit_skb()
1502 	struct virtnet_info *vi = netdev_priv(dev);  in start_xmit()  local
1504 	struct send_queue *sq = &vi->sq[qnum];  in start_xmit()
1578 static bool virtnet_send_command(struct virtnet_info *vi, u8 class, u8 cmd,  in virtnet_send_command()  argument
1585 	BUG_ON(!virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_VQ));  in virtnet_send_command()
1587 	vi->ctrl->status = ~0;  in virtnet_send_command()
1588 	vi->ctrl->hdr.class = class;  in virtnet_send_command()
1589 	vi->ctrl->hdr.cmd = cmd;  in virtnet_send_command()
1591 	sg_init_one(&hdr, &vi->ctrl->hdr, sizeof(vi->ctrl->hdr));  in virtnet_send_command()
1598 	sg_init_one(&stat, &vi->ctrl->status, sizeof(vi->ctrl->status));  in virtnet_send_command()
1602 	virtqueue_add_sgs(vi->cvq, sgs, out_num, 1, vi, GFP_ATOMIC);  in virtnet_send_command()
1604 	if (unlikely(!virtqueue_kick(vi->cvq)))  in virtnet_send_command()
1605 		return vi->ctrl->status == VIRTIO_NET_OK;  in virtnet_send_command()
1610 	while (!virtqueue_get_buf(vi->cvq, &tmp) &&  in virtnet_send_command()
1611 	       !virtqueue_is_broken(vi->cvq))  in virtnet_send_command()
1614 	return vi->ctrl->status == VIRTIO_NET_OK;  in virtnet_send_command()
1619 	struct virtnet_info *vi = netdev_priv(dev);  in virtnet_set_mac_address()  local
1620 	struct virtio_device *vdev = vi->vdev;  in virtnet_set_mac_address()
1625 	if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_STANDBY))  in virtnet_set_mac_address()
1638 		if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_MAC,  in virtnet_set_mac_address()
1667 	struct virtnet_info *vi = netdev_priv(dev);  in virtnet_stats()  local
1671 	for (i = 0; i < vi->max_queue_pairs; i++) {  in virtnet_stats()
1673 		struct receive_queue *rq = &vi->rq[i];  in virtnet_stats()
1674 		struct send_queue *sq = &vi->sq[i];  in virtnet_stats()
1702 static void virtnet_ack_link_announce(struct virtnet_info *vi)  in virtnet_ack_link_announce()  argument
1705 	if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_ANNOUNCE,  in virtnet_ack_link_announce()
1707 		dev_warn(&vi->dev->dev, "Failed to ack link announce.\n");  in virtnet_ack_link_announce()
1711 static int _virtnet_set_queues(struct virtnet_info *vi, u16 queue_pairs)  in _virtnet_set_queues()  argument
1714 	struct net_device *dev = vi->dev;  in _virtnet_set_queues()
1716 	if (!vi->has_cvq || !virtio_has_feature(vi->vdev, VIRTIO_NET_F_MQ))  in _virtnet_set_queues()
1719 	vi->ctrl->mq.virtqueue_pairs = cpu_to_virtio16(vi->vdev, queue_pairs);  in _virtnet_set_queues()
1720 	sg_init_one(&sg, &vi->ctrl->mq, sizeof(vi->ctrl->mq));  in _virtnet_set_queues()
1722 	if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_MQ,  in _virtnet_set_queues()
1728 		vi->curr_queue_pairs = queue_pairs;  in _virtnet_set_queues()
1731 			schedule_delayed_work(&vi->refill, 0);  in _virtnet_set_queues()
1737 static int virtnet_set_queues(struct virtnet_info *vi, u16 queue_pairs)  in virtnet_set_queues()  argument
1742 	err = _virtnet_set_queues(vi, queue_pairs);  in virtnet_set_queues()
1749 	struct virtnet_info *vi = netdev_priv(dev);  in virtnet_close()  local
1753 	cancel_delayed_work_sync(&vi->refill);  in virtnet_close()
1755 	for (i = 0; i < vi->max_queue_pairs; i++) {  in virtnet_close()
1756 		xdp_rxq_info_unreg(&vi->rq[i].xdp_rxq);  in virtnet_close()
1757 		napi_disable(&vi->rq[i].napi);  in virtnet_close()
1758 		virtnet_napi_tx_disable(&vi->sq[i].napi);  in virtnet_close()
1766 	struct virtnet_info *vi = netdev_priv(dev);  in virtnet_set_rx_mode()  local
1776 	if (!virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_RX))  in virtnet_set_rx_mode()
1779 	vi->ctrl->promisc = ((dev->flags & IFF_PROMISC) != 0);  in virtnet_set_rx_mode()
1780 	vi->ctrl->allmulti = ((dev->flags & IFF_ALLMULTI) != 0);  in virtnet_set_rx_mode()
1782 	sg_init_one(sg, &vi->ctrl->promisc, sizeof(vi->ctrl->promisc));  in virtnet_set_rx_mode()
1784 	if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_RX,  in virtnet_set_rx_mode()
1787 			 vi->ctrl->promisc ? "en" : "dis");  in virtnet_set_rx_mode()
1789 	sg_init_one(sg, &vi->ctrl->allmulti, sizeof(vi->ctrl->allmulti));  in virtnet_set_rx_mode()
1791 	if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_RX,  in virtnet_set_rx_mode()
1794 			 vi->ctrl->allmulti ? "en" : "dis");  in virtnet_set_rx_mode()
1808 	mac_data->entries = cpu_to_virtio32(vi->vdev, uc_count);  in virtnet_set_rx_mode()
1819 	mac_data->entries = cpu_to_virtio32(vi->vdev, mc_count);  in virtnet_set_rx_mode()
1827 	if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_MAC,  in virtnet_set_rx_mode()
1837 	struct virtnet_info *vi = netdev_priv(dev);  in virtnet_vlan_rx_add_vid()  local
1840 	vi->ctrl->vid = cpu_to_virtio16(vi->vdev, vid);  in virtnet_vlan_rx_add_vid()
1841 	sg_init_one(&sg, &vi->ctrl->vid, sizeof(vi->ctrl->vid));  in virtnet_vlan_rx_add_vid()
1843 	if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_VLAN,  in virtnet_vlan_rx_add_vid()
1852 	struct virtnet_info *vi = netdev_priv(dev);  in virtnet_vlan_rx_kill_vid()  local
1855 	vi->ctrl->vid = cpu_to_virtio16(vi->vdev, vid);  in virtnet_vlan_rx_kill_vid()
1856 	sg_init_one(&sg, &vi->ctrl->vid, sizeof(vi->ctrl->vid));  in virtnet_vlan_rx_kill_vid()
1858 	if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_VLAN,  in virtnet_vlan_rx_kill_vid()
1864 static void virtnet_clean_affinity(struct virtnet_info *vi, long hcpu)  in virtnet_clean_affinity()  argument
1868 	if (vi->affinity_hint_set) {  in virtnet_clean_affinity()
1869 		for (i = 0; i < vi->max_queue_pairs; i++) {  in virtnet_clean_affinity()
1870 			virtqueue_set_affinity(vi->rq[i].vq, NULL);  in virtnet_clean_affinity()
1871 			virtqueue_set_affinity(vi->sq[i].vq, NULL);  in virtnet_clean_affinity()
1874 		vi->affinity_hint_set = false;  in virtnet_clean_affinity()
1878 static void virtnet_set_affinity(struct virtnet_info *vi)  in virtnet_set_affinity()  argument
1888 		virtnet_clean_affinity(vi, -1);  in virtnet_set_affinity()
1893 	stride = max_t(int, num_cpu / vi->curr_queue_pairs, 1);  in virtnet_set_affinity()
1894 	stragglers = num_cpu >= vi->curr_queue_pairs ?  in virtnet_set_affinity()
1895 			num_cpu % vi->curr_queue_pairs :  in virtnet_set_affinity()
1899 	for (i = 0; i < vi->curr_queue_pairs; i++) {  in virtnet_set_affinity()
1907 		virtqueue_set_affinity(vi->rq[i].vq, mask);  in virtnet_set_affinity()
1908 		virtqueue_set_affinity(vi->sq[i].vq, mask);  in virtnet_set_affinity()
1909 		__netif_set_xps_queue(vi->dev, cpumask_bits(mask), i, false);  in virtnet_set_affinity()
1913 	vi->affinity_hint_set = true;  in virtnet_set_affinity()
1919 	struct virtnet_info *vi = hlist_entry_safe(node, struct virtnet_info,  in virtnet_cpu_online()  local
1921 	virtnet_set_affinity(vi);  in virtnet_cpu_online()
1927 	struct virtnet_info *vi = hlist_entry_safe(node, struct virtnet_info,  in virtnet_cpu_dead()  local
1929 	virtnet_set_affinity(vi);  in virtnet_cpu_dead()
1935 	struct virtnet_info *vi = hlist_entry_safe(node, struct virtnet_info,  in virtnet_cpu_down_prep()  local
1938 	virtnet_clean_affinity(vi, cpu);  in virtnet_cpu_down_prep()
1944 static int virtnet_cpu_notif_add(struct virtnet_info *vi)  in virtnet_cpu_notif_add()  argument
1948 	ret = cpuhp_state_add_instance_nocalls(virtionet_online, &vi->node);  in virtnet_cpu_notif_add()
1952 					       &vi->node_dead);  in virtnet_cpu_notif_add()
1955 	cpuhp_state_remove_instance_nocalls(virtionet_online, &vi->node);  in virtnet_cpu_notif_add()
1959 static void virtnet_cpu_notif_remove(struct virtnet_info *vi)  in virtnet_cpu_notif_remove()  argument
1961 	cpuhp_state_remove_instance_nocalls(virtionet_online, &vi->node);  in virtnet_cpu_notif_remove()
1963 					    &vi->node_dead);  in virtnet_cpu_notif_remove()
1969 	struct virtnet_info *vi = netdev_priv(dev);  in virtnet_get_ringparam()  local
1971 	ring->rx_max_pending = virtqueue_get_vring_size(vi->rq[0].vq);  in virtnet_get_ringparam()
1972 	ring->tx_max_pending = virtqueue_get_vring_size(vi->sq[0].vq);  in virtnet_get_ringparam()
1981 	struct virtnet_info *vi = netdev_priv(dev);  in virtnet_get_drvinfo()  local
1982 	struct virtio_device *vdev = vi->vdev;  in virtnet_get_drvinfo()
1994 	struct virtnet_info *vi = netdev_priv(dev);  in virtnet_set_channels()  local
2004 	if (queue_pairs > vi->max_queue_pairs || queue_pairs == 0)  in virtnet_set_channels()
2011 	if (vi->rq[0].xdp_prog)  in virtnet_set_channels()
2015 	err = _virtnet_set_queues(vi, queue_pairs);  in virtnet_set_channels()
2020 		virtnet_set_affinity(vi);  in virtnet_set_channels()
2029 	struct virtnet_info *vi = netdev_priv(dev);  in virtnet_get_strings()  local
2035 		for (i = 0; i < vi->curr_queue_pairs; i++) {  in virtnet_get_strings()
2043 		for (i = 0; i < vi->curr_queue_pairs; i++) {  in virtnet_get_strings()
2056 	struct virtnet_info *vi = netdev_priv(dev);  in virtnet_get_sset_count()  local
2060 		return vi->curr_queue_pairs * (VIRTNET_RQ_STATS_LEN +  in virtnet_get_sset_count()
2070 	struct virtnet_info *vi = netdev_priv(dev);  in virtnet_get_ethtool_stats()  local
2075 	for (i = 0; i < vi->curr_queue_pairs; i++) {  in virtnet_get_ethtool_stats()
2076 		struct receive_queue *rq = &vi->rq[i];  in virtnet_get_ethtool_stats()
2089 	for (i = 0; i < vi->curr_queue_pairs; i++) {  in virtnet_get_ethtool_stats()
2090 		struct send_queue *sq = &vi->sq[i];  in virtnet_get_ethtool_stats()
2107 	struct virtnet_info *vi = netdev_priv(dev);  in virtnet_get_channels()  local
2109 	channels->combined_count = vi->curr_queue_pairs;  in virtnet_get_channels()
2110 	channels->max_combined = vi->max_queue_pairs;  in virtnet_get_channels()
2146 	struct virtnet_info *vi = netdev_priv(dev);  in virtnet_set_link_ksettings()  local
2155 	vi->speed = speed;  in virtnet_set_link_ksettings()
2156 	vi->duplex = cmd->base.duplex;  in virtnet_set_link_ksettings()
2164 	struct virtnet_info *vi = netdev_priv(dev);  in virtnet_get_link_ksettings()  local
2166 	cmd->base.speed = vi->speed;  in virtnet_get_link_ksettings()
2167 	cmd->base.duplex = vi->duplex;  in virtnet_get_link_ksettings()
2175 	struct virtnet_info *vi = netdev_priv(dev);  in virtnet_init_settings()  local
2177 	vi->speed = SPEED_UNKNOWN;  in virtnet_init_settings()
2178 	vi->duplex = DUPLEX_UNKNOWN;  in virtnet_init_settings()
2181 static void virtnet_update_settings(struct virtnet_info *vi)  in virtnet_update_settings()  argument
2186 	if (!virtio_has_feature(vi->vdev, VIRTIO_NET_F_SPEED_DUPLEX))  in virtnet_update_settings()
2189 	speed = virtio_cread32(vi->vdev, offsetof(struct virtio_net_config,  in virtnet_update_settings()
2192 		vi->speed = speed;  in virtnet_update_settings()
2193 	duplex = virtio_cread8(vi->vdev, offsetof(struct virtio_net_config,  in virtnet_update_settings()
2196 		vi->duplex = duplex;  in virtnet_update_settings()
2215 	struct virtnet_info *vi = vdev->priv;  in virtnet_freeze_down()  local
2219 	flush_work(&vi->config_work);  in virtnet_freeze_down()
2221 	netif_tx_lock_bh(vi->dev);  in virtnet_freeze_down()
2222 	netif_device_detach(vi->dev);  in virtnet_freeze_down()
2223 	netif_tx_unlock_bh(vi->dev);  in virtnet_freeze_down()
2224 	cancel_delayed_work_sync(&vi->refill);  in virtnet_freeze_down()
2226 	if (netif_running(vi->dev)) {  in virtnet_freeze_down()
2227 		for (i = 0; i < vi->max_queue_pairs; i++) {  in virtnet_freeze_down()
2228 			napi_disable(&vi->rq[i].napi);  in virtnet_freeze_down()
2229 			virtnet_napi_tx_disable(&vi->sq[i].napi);  in virtnet_freeze_down()
2234 static int init_vqs(struct virtnet_info *vi);
2238 	struct virtnet_info *vi = vdev->priv;  in virtnet_restore_up()  local
2241 	err = init_vqs(vi);  in virtnet_restore_up()
2247 	if (netif_running(vi->dev)) {  in virtnet_restore_up()
2248 		for (i = 0; i < vi->curr_queue_pairs; i++)  in virtnet_restore_up()
2249 			if (!try_fill_recv(vi, &vi->rq[i], GFP_KERNEL))  in virtnet_restore_up()
2250 				schedule_delayed_work(&vi->refill, 0);  in virtnet_restore_up()
2252 		for (i = 0; i < vi->max_queue_pairs; i++) {  in virtnet_restore_up()
2253 			virtnet_napi_enable(vi->rq[i].vq, &vi->rq[i].napi);  in virtnet_restore_up()
2254 			virtnet_napi_tx_enable(vi, vi->sq[i].vq,  in virtnet_restore_up()
2255 					       &vi->sq[i].napi);  in virtnet_restore_up()
2259 	netif_tx_lock_bh(vi->dev);  in virtnet_restore_up()
2260 	netif_device_attach(vi->dev);  in virtnet_restore_up()
2261 	netif_tx_unlock_bh(vi->dev);  in virtnet_restore_up()
2265 static int virtnet_set_guest_offloads(struct virtnet_info *vi, u64 offloads)  in virtnet_set_guest_offloads()  argument
2268 	vi->ctrl->offloads = cpu_to_virtio64(vi->vdev, offloads);  in virtnet_set_guest_offloads()
2270 	sg_init_one(&sg, &vi->ctrl->offloads, sizeof(vi->ctrl->offloads));  in virtnet_set_guest_offloads()
2272 	if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_GUEST_OFFLOADS,  in virtnet_set_guest_offloads()
2274 		dev_warn(&vi->dev->dev, "Fail to set guest offload. \n");  in virtnet_set_guest_offloads()
2281 static int virtnet_clear_guest_offloads(struct virtnet_info *vi)  in virtnet_clear_guest_offloads()  argument
2285 	if (!vi->guest_offloads)  in virtnet_clear_guest_offloads()
2288 	if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_CSUM))  in virtnet_clear_guest_offloads()
2291 	return virtnet_set_guest_offloads(vi, offloads);  in virtnet_clear_guest_offloads()
2294 static int virtnet_restore_guest_offloads(struct virtnet_info *vi)  in virtnet_restore_guest_offloads()  argument
2296 	u64 offloads = vi->guest_offloads;  in virtnet_restore_guest_offloads()
2298 	if (!vi->guest_offloads)  in virtnet_restore_guest_offloads()
2300 	if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_CSUM))  in virtnet_restore_guest_offloads()
2303 	return virtnet_set_guest_offloads(vi, offloads);  in virtnet_restore_guest_offloads()
2310 	struct virtnet_info *vi = netdev_priv(dev);  in virtnet_xdp_set()  local
2315 	if (!virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_GUEST_OFFLOADS)  in virtnet_xdp_set()
2316 	    && (virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_TSO4) ||  in virtnet_xdp_set()
2317 	        virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_TSO6) ||  in virtnet_xdp_set()
2318 	        virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_ECN) ||  in virtnet_xdp_set()
2319 		virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_UFO))) {  in virtnet_xdp_set()
2324 	if (vi->mergeable_rx_bufs && !vi->any_header_sg) {  in virtnet_xdp_set()
2335 	curr_qp = vi->curr_queue_pairs - vi->xdp_queue_pairs;  in virtnet_xdp_set()
2340 	if (curr_qp + xdp_qp > vi->max_queue_pairs) {  in virtnet_xdp_set()
2343 			    curr_qp + xdp_qp, vi->max_queue_pairs);  in virtnet_xdp_set()
2348 		prog = bpf_prog_add(prog, vi->max_queue_pairs - 1);  in virtnet_xdp_set()
2355 		for (i = 0; i < vi->max_queue_pairs; i++)  in virtnet_xdp_set()
2356 			napi_disable(&vi->rq[i].napi);  in virtnet_xdp_set()
2359 	err = _virtnet_set_queues(vi, curr_qp + xdp_qp);  in virtnet_xdp_set()
2362 	vi->xdp_queue_pairs = xdp_qp;  in virtnet_xdp_set()
2364 	for (i = 0; i < vi->max_queue_pairs; i++) {  in virtnet_xdp_set()
2365 		old_prog = rtnl_dereference(vi->rq[i].xdp_prog);  in virtnet_xdp_set()
2366 		rcu_assign_pointer(vi->rq[i].xdp_prog, prog);  in virtnet_xdp_set()
2369 				virtnet_clear_guest_offloads(vi);  in virtnet_xdp_set()
2371 				virtnet_restore_guest_offloads(vi);  in virtnet_xdp_set()
2376 			virtnet_napi_enable(vi->rq[i].vq, &vi->rq[i].napi);  in virtnet_xdp_set()
2382 	for (i = 0; i < vi->max_queue_pairs; i++)  in virtnet_xdp_set()
2383 		virtnet_napi_enable(vi->rq[i].vq, &vi->rq[i].napi);  in virtnet_xdp_set()
2385 		bpf_prog_sub(prog, vi->max_queue_pairs - 1);  in virtnet_xdp_set()
2391 	struct virtnet_info *vi = netdev_priv(dev);  in virtnet_xdp_query()  local
2395 	for (i = 0; i < vi->max_queue_pairs; i++) {  in virtnet_xdp_query()
2396 		xdp_prog = rtnl_dereference(vi->rq[i].xdp_prog);  in virtnet_xdp_query()
2419 	struct virtnet_info *vi = netdev_priv(dev);  in virtnet_get_phys_port_name()  local
2422 	if (!virtio_has_feature(vi->vdev, VIRTIO_NET_F_STANDBY))  in virtnet_get_phys_port_name()
2450 	struct virtnet_info *vi =  in virtnet_config_changed_work()  local
2454 	if (virtio_cread_feature(vi->vdev, VIRTIO_NET_F_STATUS,  in virtnet_config_changed_work()
2459 		netdev_notify_peers(vi->dev);  in virtnet_config_changed_work()
2460 		virtnet_ack_link_announce(vi);  in virtnet_config_changed_work()
2466 	if (vi->status == v)  in virtnet_config_changed_work()
2469 	vi->status = v;  in virtnet_config_changed_work()
2471 	if (vi->status & VIRTIO_NET_S_LINK_UP) {  in virtnet_config_changed_work()
2472 		virtnet_update_settings(vi);  in virtnet_config_changed_work()
2473 		netif_carrier_on(vi->dev);  in virtnet_config_changed_work()
2474 		netif_tx_wake_all_queues(vi->dev);  in virtnet_config_changed_work()
2476 		netif_carrier_off(vi->dev);  in virtnet_config_changed_work()
2477 		netif_tx_stop_all_queues(vi->dev);  in virtnet_config_changed_work()
2483 	struct virtnet_info *vi = vdev->priv;  in virtnet_config_changed()  local
2485 	schedule_work(&vi->config_work);  in virtnet_config_changed()
2488 static void virtnet_free_queues(struct virtnet_info *vi)  in virtnet_free_queues()  argument
2492 	for (i = 0; i < vi->max_queue_pairs; i++) {  in virtnet_free_queues()
2493 		napi_hash_del(&vi->rq[i].napi);  in virtnet_free_queues()
2494 		netif_napi_del(&vi->rq[i].napi);  in virtnet_free_queues()
2495 		netif_napi_del(&vi->sq[i].napi);  in virtnet_free_queues()
2503 	kfree(vi->rq);  in virtnet_free_queues()
2504 	kfree(vi->sq);  in virtnet_free_queues()
2505 	kfree(vi->ctrl);  in virtnet_free_queues()
2508 static void _free_receive_bufs(struct virtnet_info *vi)  in _free_receive_bufs()  argument
2513 	for (i = 0; i < vi->max_queue_pairs; i++) {  in _free_receive_bufs()
2514 		while (vi->rq[i].pages)  in _free_receive_bufs()
2515 			__free_pages(get_a_page(&vi->rq[i], GFP_KERNEL), 0);  in _free_receive_bufs()
2517 		old_prog = rtnl_dereference(vi->rq[i].xdp_prog);  in _free_receive_bufs()
2518 		RCU_INIT_POINTER(vi->rq[i].xdp_prog, NULL);  in _free_receive_bufs()
2524 static void free_receive_bufs(struct virtnet_info *vi)  in free_receive_bufs()  argument
2527 	_free_receive_bufs(vi);  in free_receive_bufs()
2531 static void free_receive_page_frags(struct virtnet_info *vi)  in free_receive_page_frags()  argument
2534 	for (i = 0; i < vi->max_queue_pairs; i++)  in free_receive_page_frags()
2535 		if (vi->rq[i].alloc_frag.page)  in free_receive_page_frags()
2536 			put_page(vi->rq[i].alloc_frag.page);  in free_receive_page_frags()
2539 static bool is_xdp_raw_buffer_queue(struct virtnet_info *vi, int q)  in is_xdp_raw_buffer_queue()  argument
2541 	if (q < (vi->curr_queue_pairs - vi->xdp_queue_pairs))  in is_xdp_raw_buffer_queue()
2543 	else if (q < vi->curr_queue_pairs)  in is_xdp_raw_buffer_queue()
2549 static void free_unused_bufs(struct virtnet_info *vi)  in free_unused_bufs()  argument
2554 	for (i = 0; i < vi->max_queue_pairs; i++) {  in free_unused_bufs()
2555 		struct virtqueue *vq = vi->sq[i].vq;  in free_unused_bufs()
2557 			if (!is_xdp_raw_buffer_queue(vi, i))  in free_unused_bufs()
2564 	for (i = 0; i < vi->max_queue_pairs; i++) {  in free_unused_bufs()
2565 		struct virtqueue *vq = vi->rq[i].vq;  in free_unused_bufs()
2568 			if (vi->mergeable_rx_bufs) {  in free_unused_bufs()
2570 			} else if (vi->big_packets) {  in free_unused_bufs()
2571 				give_pages(&vi->rq[i], buf);  in free_unused_bufs()
2579 static void virtnet_del_vqs(struct virtnet_info *vi)  in virtnet_del_vqs()  argument
2581 	struct virtio_device *vdev = vi->vdev;  in virtnet_del_vqs()
2583 	virtnet_clean_affinity(vi, -1);  in virtnet_del_vqs()
2587 	virtnet_free_queues(vi);  in virtnet_del_vqs()
2594 static unsigned int mergeable_min_buf_len(struct virtnet_info *vi, struct virtqueue *vq)  in mergeable_min_buf_len()  argument
2598 	unsigned int packet_len = vi->big_packets ? IP_MAX_MTU : vi->dev->max_mtu;  in mergeable_min_buf_len()
2606 static int virtnet_find_vqs(struct virtnet_info *vi)  in virtnet_find_vqs()  argument
2619 	total_vqs = vi->max_queue_pairs * 2 +  in virtnet_find_vqs()
2620 		    virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_VQ);  in virtnet_find_vqs()
2632 	if (!vi->big_packets || vi->mergeable_rx_bufs) {  in virtnet_find_vqs()
2641 	if (vi->has_cvq) {  in virtnet_find_vqs()
2647 	for (i = 0; i < vi->max_queue_pairs; i++) {  in virtnet_find_vqs()
2650 		sprintf(vi->rq[i].name, "input.%d", i);  in virtnet_find_vqs()
2651 		sprintf(vi->sq[i].name, "output.%d", i);  in virtnet_find_vqs()
2652 		names[rxq2vq(i)] = vi->rq[i].name;  in virtnet_find_vqs()
2653 		names[txq2vq(i)] = vi->sq[i].name;  in virtnet_find_vqs()
2658 	ret = vi->vdev->config->find_vqs(vi->vdev, total_vqs, vqs, callbacks,  in virtnet_find_vqs()
2663 	if (vi->has_cvq) {  in virtnet_find_vqs()
2664 		vi->cvq = vqs[total_vqs - 1];  in virtnet_find_vqs()
2665 		if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_VLAN))  in virtnet_find_vqs()
2666 			vi->dev->features |= NETIF_F_HW_VLAN_CTAG_FILTER;  in virtnet_find_vqs()
2669 	for (i = 0; i < vi->max_queue_pairs; i++) {  in virtnet_find_vqs()
2670 		vi->rq[i].vq = vqs[rxq2vq(i)];  in virtnet_find_vqs()
2671 		vi->rq[i].min_buf_len = mergeable_min_buf_len(vi, vi->rq[i].vq);  in virtnet_find_vqs()
2672 		vi->sq[i].vq = vqs[txq2vq(i)];  in virtnet_find_vqs()
2690 static int virtnet_alloc_queues(struct virtnet_info *vi)  in virtnet_alloc_queues()  argument
2694 	vi->ctrl = kzalloc(sizeof(*vi->ctrl), GFP_KERNEL);  in virtnet_alloc_queues()
2695 	if (!vi->ctrl)  in virtnet_alloc_queues()
2697 	vi->sq = kcalloc(vi->max_queue_pairs, sizeof(*vi->sq), GFP_KERNEL);  in virtnet_alloc_queues()
2698 	if (!vi->sq)  in virtnet_alloc_queues()
2700 	vi->rq = kcalloc(vi->max_queue_pairs, sizeof(*vi->rq), GFP_KERNEL);  in virtnet_alloc_queues()
2701 	if (!vi->rq)  in virtnet_alloc_queues()
2704 	INIT_DELAYED_WORK(&vi->refill, refill_work);  in virtnet_alloc_queues()
2705 	for (i = 0; i < vi->max_queue_pairs; i++) {  in virtnet_alloc_queues()
2706 		vi->rq[i].pages = NULL;  in virtnet_alloc_queues()
2707 		netif_napi_add(vi->dev, &vi->rq[i].napi, virtnet_poll,  in virtnet_alloc_queues()
2709 		netif_tx_napi_add(vi->dev, &vi->sq[i].napi, virtnet_poll_tx,  in virtnet_alloc_queues()
2712 		sg_init_table(vi->rq[i].sg, ARRAY_SIZE(vi->rq[i].sg));  in virtnet_alloc_queues()
2713 		ewma_pkt_len_init(&vi->rq[i].mrg_avg_pkt_len);  in virtnet_alloc_queues()
2714 		sg_init_table(vi->sq[i].sg, ARRAY_SIZE(vi->sq[i].sg));  in virtnet_alloc_queues()
2716 		u64_stats_init(&vi->rq[i].stats.syncp);  in virtnet_alloc_queues()
2717 		u64_stats_init(&vi->sq[i].stats.syncp);  in virtnet_alloc_queues()
2723 	kfree(vi->sq);  in virtnet_alloc_queues()
2725 	kfree(vi->ctrl);  in virtnet_alloc_queues()
2730 static int init_vqs(struct virtnet_info *vi)  in init_vqs()  argument
2735 	ret = virtnet_alloc_queues(vi);  in init_vqs()
2739 	ret = virtnet_find_vqs(vi);  in init_vqs()
2744 	virtnet_set_affinity(vi);  in init_vqs()
2750 	virtnet_free_queues(vi);  in init_vqs()
2759 	struct virtnet_info *vi = netdev_priv(queue->dev);  in mergeable_rx_buffer_size_show()  local
2761 	unsigned int headroom = virtnet_get_headroom(vi);  in mergeable_rx_buffer_size_show()
2765 	BUG_ON(queue_index >= vi->max_queue_pairs);  in mergeable_rx_buffer_size_show()
2766 	avg = &vi->rq[queue_index].mrg_avg_pkt_len;  in mergeable_rx_buffer_size_show()
2768 		       get_mergeable_buf_len(&vi->rq[queue_index], avg,  in mergeable_rx_buffer_size_show()
2849 	struct virtnet_info *vi;  in virtnet_probe()  local
2920 	vi = netdev_priv(dev);  in virtnet_probe()
2921 	vi->dev = dev;  in virtnet_probe()
2922 	vi->vdev = vdev;  in virtnet_probe()
2923 	vdev->priv = vi;  in virtnet_probe()
2925 	INIT_WORK(&vi->config_work, virtnet_config_changed_work);  in virtnet_probe()
2932 		vi->big_packets = true;  in virtnet_probe()
2935 		vi->mergeable_rx_bufs = true;  in virtnet_probe()
2939 		vi->hdr_len = sizeof(struct virtio_net_hdr_mrg_rxbuf);  in virtnet_probe()
2941 		vi->hdr_len = sizeof(struct virtio_net_hdr);  in virtnet_probe()
2945 		vi->any_header_sg = true;  in virtnet_probe()
2948 		vi->has_cvq = true;  in virtnet_probe()
2968 			vi->big_packets = true;  in virtnet_probe()
2971 	if (vi->any_header_sg)  in virtnet_probe()
2972 		dev->needed_headroom = vi->hdr_len;  in virtnet_probe()
2976 		vi->curr_queue_pairs = max_queue_pairs;  in virtnet_probe()
2978 		vi->curr_queue_pairs = num_online_cpus();  in virtnet_probe()
2979 	vi->max_queue_pairs = max_queue_pairs;  in virtnet_probe()
2982 	err = init_vqs(vi);  in virtnet_probe()
2987 	if (vi->mergeable_rx_bufs)  in virtnet_probe()
2990 	netif_set_real_num_tx_queues(dev, vi->curr_queue_pairs);  in virtnet_probe()
2991 	netif_set_real_num_rx_queues(dev, vi->curr_queue_pairs);  in virtnet_probe()
2996 		vi->failover = net_failover_create(vi->dev);  in virtnet_probe()
2997 		if (IS_ERR(vi->failover)) {  in virtnet_probe()
2998 			err = PTR_ERR(vi->failover);  in virtnet_probe()
3011 	err = virtnet_cpu_notif_add(vi);  in virtnet_probe()
3017 	virtnet_set_queues(vi, vi->curr_queue_pairs);  in virtnet_probe()
3022 	if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_STATUS)) {  in virtnet_probe()
3023 		schedule_work(&vi->config_work);  in virtnet_probe()
3025 		vi->status = VIRTIO_NET_S_LINK_UP;  in virtnet_probe()
3026 		virtnet_update_settings(vi);  in virtnet_probe()
3031 		if (virtio_has_feature(vi->vdev, guest_offloads[i]))  in virtnet_probe()
3032 			set_bit(guest_offloads[i], &vi->guest_offloads);  in virtnet_probe()
3040 	vi->vdev->config->reset(vdev);  in virtnet_probe()
3044 	net_failover_destroy(vi->failover);  in virtnet_probe()
3046 	cancel_delayed_work_sync(&vi->refill);  in virtnet_probe()
3047 	free_receive_page_frags(vi);  in virtnet_probe()
3048 	virtnet_del_vqs(vi);  in virtnet_probe()
3054 static void remove_vq_common(struct virtnet_info *vi)  in remove_vq_common()  argument
3056 	vi->vdev->config->reset(vi->vdev);  in remove_vq_common()
3059 	free_unused_bufs(vi);  in remove_vq_common()
3061 	free_receive_bufs(vi);  in remove_vq_common()
3063 	free_receive_page_frags(vi);  in remove_vq_common()
3065 	virtnet_del_vqs(vi);  in remove_vq_common()
3070 	struct virtnet_info *vi = vdev->priv;  in virtnet_remove()  local
3072 	virtnet_cpu_notif_remove(vi);  in virtnet_remove()
3075 	flush_work(&vi->config_work);  in virtnet_remove()
3077 	unregister_netdev(vi->dev);  in virtnet_remove()
3079 	net_failover_destroy(vi->failover);  in virtnet_remove()
3081 	remove_vq_common(vi);  in virtnet_remove()
3083 	free_netdev(vi->dev);  in virtnet_remove()
3088 	struct virtnet_info *vi = vdev->priv;  in virtnet_freeze()  local
3090 	virtnet_cpu_notif_remove(vi);  in virtnet_freeze()
3092 	remove_vq_common(vi);  in virtnet_freeze()
3099 	struct virtnet_info *vi = vdev->priv;  in virtnet_restore()  local
3105 	virtnet_set_queues(vi, vi->curr_queue_pairs);  in virtnet_restore()
3107 	err = virtnet_cpu_notif_add(vi);  in virtnet_restore()