339 	struct virtnet_info *vi = vq->vdev->priv;  in skb_xmit_done()  local
340 	struct napi_struct *napi = &vi->sq[vq2txq(vq)].napi;  in skb_xmit_done()
349 		netif_wake_subqueue(vi->dev, vq2txq(vq));  in skb_xmit_done()
370 static struct sk_buff *page_to_skb(struct virtnet_info *vi,  in page_to_skb()  argument
390 	hdr_len = vi->hdr_len;  in page_to_skb()
391 	if (vi->mergeable_rx_bufs)  in page_to_skb()
411 	if (vi->mergeable_rx_bufs) {  in page_to_skb()
446 static int __virtnet_xdp_xmit_one(struct virtnet_info *vi,  in __virtnet_xdp_xmit_one()  argument
457 	if (unlikely(xdpf->headroom < vi->hdr_len))  in __virtnet_xdp_xmit_one()
461 	xdpf->data -= vi->hdr_len;  in __virtnet_xdp_xmit_one()
464 	memset(hdr, 0, vi->hdr_len);  in __virtnet_xdp_xmit_one()
465 	xdpf->len   += vi->hdr_len;  in __virtnet_xdp_xmit_one()
477 static struct send_queue *virtnet_xdp_sq(struct virtnet_info *vi)  in virtnet_xdp_sq()  argument
481 	qp = vi->curr_queue_pairs - vi->xdp_queue_pairs + smp_processor_id();  in virtnet_xdp_sq()
482 	return &vi->sq[qp];  in virtnet_xdp_sq()
488 	struct virtnet_info *vi = netdev_priv(dev);  in virtnet_xdp_xmit()  local
489 	struct receive_queue *rq = vi->rq;  in virtnet_xdp_xmit()
508 	sq = virtnet_xdp_sq(vi);  in virtnet_xdp_xmit()
535 		err = __virtnet_xdp_xmit_one(vi, sq, xdpf);  in virtnet_xdp_xmit()
559 static unsigned int virtnet_get_headroom(struct virtnet_info *vi)  in virtnet_get_headroom()  argument
561 	return vi->xdp_queue_pairs ? VIRTIO_XDP_HEADROOM : 0;  in virtnet_get_headroom()
629 				     struct virtnet_info *vi,  in receive_small()  argument
640 	unsigned int headroom = vi->hdr_len + header_offset;  in receive_small()
648 	len -= vi->hdr_len;  in receive_small()
663 		if (unlikely(xdp_headroom < virtnet_get_headroom(vi))) {  in receive_small()
665 			unsigned int tlen = len + vi->hdr_len;  in receive_small()
668 			xdp_headroom = virtnet_get_headroom(vi);  in receive_small()
670 			headroom = vi->hdr_len + header_offset;  in receive_small()
684 		xdp.data_hard_start = buf + VIRTNET_RX_PAD + vi->hdr_len;  in receive_small()
706 				trace_xdp_exception(vi->dev, xdp_prog, act);  in receive_small()
724 			trace_xdp_exception(vi->dev, xdp_prog, act);  in receive_small()
740 		memcpy(skb_vnet_hdr(skb), buf, vi->hdr_len);  in receive_small()
756 				   struct virtnet_info *vi,  in receive_big()  argument
763 	struct sk_buff *skb = page_to_skb(vi, rq, page, 0, len,  in receive_big()
766 	stats->bytes += len - vi->hdr_len;  in receive_big()
779 					 struct virtnet_info *vi,  in receive_mergeable()  argument
788 	u16 num_buf = virtio16_to_cpu(vi->vdev, hdr->num_buffers);  in receive_mergeable()
798 	stats->bytes += len - vi->hdr_len;  in receive_mergeable()
823 			     headroom < virtnet_get_headroom(vi))) {  in receive_mergeable()
840 		xdp.data_hard_start = data - VIRTIO_XDP_HEADROOM + vi->hdr_len;  in receive_mergeable()
841 		xdp.data = data + vi->hdr_len;  in receive_mergeable()
843 		xdp.data_end = xdp.data + (len - vi->hdr_len);  in receive_mergeable()
856 					page_address(xdp_page) - vi->hdr_len;  in receive_mergeable()
861 			len = xdp.data_end - xdp.data + vi->hdr_len;  in receive_mergeable()
866 				head_skb = page_to_skb(vi, rq, xdp_page,  in receive_mergeable()
879 				trace_xdp_exception(vi->dev, xdp_prog, act);  in receive_mergeable()
906 			trace_xdp_exception(vi->dev, xdp_prog, act);  in receive_mergeable()
924 	head_skb = page_to_skb(vi, rq, page, offset, len, truesize, !xdp_prog);  in receive_mergeable()
936 				 virtio16_to_cpu(vi->vdev,  in receive_mergeable()
1010 static void receive_buf(struct virtnet_info *vi, struct receive_queue *rq,  in receive_buf()  argument
1015 	struct net_device *dev = vi->dev;  in receive_buf()
1019 	if (unlikely(len < vi->hdr_len + ETH_HLEN)) {  in receive_buf()
1022 		if (vi->mergeable_rx_bufs) {  in receive_buf()
1024 		} else if (vi->big_packets) {  in receive_buf()
1032 	if (vi->mergeable_rx_bufs)  in receive_buf()
1033 		skb = receive_mergeable(dev, vi, rq, buf, ctx, len, xdp_xmit,  in receive_buf()
1035 	else if (vi->big_packets)  in receive_buf()
1036 		skb = receive_big(dev, vi, rq, buf, len, stats);  in receive_buf()
1038 		skb = receive_small(dev, vi, rq, buf, ctx, len, xdp_xmit, stats);  in receive_buf()
1049 				  virtio_is_little_endian(vi->vdev))) {  in receive_buf()
1074 static int add_recvbuf_small(struct virtnet_info *vi, struct receive_queue *rq,  in add_recvbuf_small()  argument
1079 	unsigned int xdp_headroom = virtnet_get_headroom(vi);  in add_recvbuf_small()
1081 	int len = vi->hdr_len + VIRTNET_RX_PAD + GOOD_PACKET_LEN + xdp_headroom;  in add_recvbuf_small()
1093 		    vi->hdr_len + GOOD_PACKET_LEN);  in add_recvbuf_small()
1100 static int add_recvbuf_big(struct virtnet_info *vi, struct receive_queue *rq,  in add_recvbuf_big()  argument
1133 	sg_set_buf(&rq->sg[0], p, vi->hdr_len);  in add_recvbuf_big()
1165 static int add_recvbuf_mergeable(struct virtnet_info *vi,  in add_recvbuf_mergeable()  argument
1169 	unsigned int headroom = virtnet_get_headroom(vi);  in add_recvbuf_mergeable()
1215 static bool try_fill_recv(struct virtnet_info *vi, struct receive_queue *rq,  in try_fill_recv()  argument
1222 		if (vi->mergeable_rx_bufs)  in try_fill_recv()
1223 			err = add_recvbuf_mergeable(vi, rq, gfp);  in try_fill_recv()
1224 		else if (vi->big_packets)  in try_fill_recv()
1225 			err = add_recvbuf_big(vi, rq, gfp);  in try_fill_recv()
1227 			err = add_recvbuf_small(vi, rq, gfp);  in try_fill_recv()
1244 	struct virtnet_info *vi = rvq->vdev->priv;  in skb_recv_done()  local
1245 	struct receive_queue *rq = &vi->rq[vq2rxq(rvq)];  in skb_recv_done()
1263 static void virtnet_napi_tx_enable(struct virtnet_info *vi,  in virtnet_napi_tx_enable()  argument
1273 	if (!vi->affinity_hint_set) {  in virtnet_napi_tx_enable()
1289 	struct virtnet_info *vi =  in refill_work()  local
1294 	for (i = 0; i < vi->curr_queue_pairs; i++) {  in refill_work()
1295 		struct receive_queue *rq = &vi->rq[i];  in refill_work()
1298 		still_empty = !try_fill_recv(vi, rq, GFP_KERNEL);  in refill_work()
1305 			schedule_delayed_work(&vi->refill, HZ/2);  in refill_work()
1312 	struct virtnet_info *vi = rq->vq->vdev->priv;  in virtnet_receive()  local
1318 	if (!vi->big_packets || vi->mergeable_rx_bufs) {  in virtnet_receive()
1323 			receive_buf(vi, rq, buf, len, ctx, xdp_xmit, &stats);  in virtnet_receive()
1329 			receive_buf(vi, rq, buf, len, NULL, xdp_xmit, &stats);  in virtnet_receive()
1335 		if (!try_fill_recv(vi, rq, GFP_ATOMIC))  in virtnet_receive()
1336 			schedule_delayed_work(&vi->refill, 0);  in virtnet_receive()
1388 static bool is_xdp_raw_buffer_queue(struct virtnet_info *vi, int q)  in is_xdp_raw_buffer_queue()  argument
1390 	if (q < (vi->curr_queue_pairs - vi->xdp_queue_pairs))  in is_xdp_raw_buffer_queue()
1392 	else if (q < vi->curr_queue_pairs)  in is_xdp_raw_buffer_queue()
1400 	struct virtnet_info *vi = rq->vq->vdev->priv;  in virtnet_poll_cleantx()  local
1402 	struct send_queue *sq = &vi->sq[index];  in virtnet_poll_cleantx()
1403 	struct netdev_queue *txq = netdev_get_tx_queue(vi->dev, index);  in virtnet_poll_cleantx()
1405 	if (!sq->napi.weight || is_xdp_raw_buffer_queue(vi, index))  in virtnet_poll_cleantx()
1421 	struct virtnet_info *vi = rq->vq->vdev->priv;  in virtnet_poll()  local
1438 		sq = virtnet_xdp_sq(vi);  in virtnet_poll()
1451 	struct virtnet_info *vi = netdev_priv(dev);  in virtnet_open()  local
1454 	for (i = 0; i < vi->max_queue_pairs; i++) {  in virtnet_open()
1455 		if (i < vi->curr_queue_pairs)  in virtnet_open()
1457 			if (!try_fill_recv(vi, &vi->rq[i], GFP_KERNEL))  in virtnet_open()
1458 				schedule_delayed_work(&vi->refill, 0);  in virtnet_open()
1460 		err = xdp_rxq_info_reg(&vi->rq[i].xdp_rxq, dev, i);  in virtnet_open()
1464 		err = xdp_rxq_info_reg_mem_model(&vi->rq[i].xdp_rxq,  in virtnet_open()
1467 			xdp_rxq_info_unreg(&vi->rq[i].xdp_rxq);  in virtnet_open()
1471 		virtnet_napi_enable(vi->rq[i].vq, &vi->rq[i].napi);  in virtnet_open()
1472 		virtnet_napi_tx_enable(vi, vi->sq[i].vq, &vi->sq[i].napi);  in virtnet_open()
1481 	struct virtnet_info *vi = sq->vq->vdev->priv;  in virtnet_poll_tx()  local
1485 	if (unlikely(is_xdp_raw_buffer_queue(vi, index))) {  in virtnet_poll_tx()
1491 	txq = netdev_get_tx_queue(vi->dev, index);  in virtnet_poll_tx()
1508 	struct virtnet_info *vi = sq->vq->vdev->priv;  in xmit_skb()  local
1510 	unsigned hdr_len = vi->hdr_len;  in xmit_skb()
1513 	pr_debug("%s: xmit %p %pM\n", vi->dev->name, skb, dest);  in xmit_skb()
1515 	can_push = vi->any_header_sg &&  in xmit_skb()
1526 				    virtio_is_little_endian(vi->vdev), false,  in xmit_skb()
1530 	if (vi->mergeable_rx_bufs)  in xmit_skb()
1553 	struct virtnet_info *vi = netdev_priv(dev);  in start_xmit()  local
1555 	struct send_queue *sq = &vi->sq[qnum];  in start_xmit()
1630 static bool virtnet_send_command(struct virtnet_info *vi, u8 class, u8 cmd,  in virtnet_send_command()  argument
1637 	BUG_ON(!virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_VQ));  in virtnet_send_command()
1639 	vi->ctrl->status = ~0;  in virtnet_send_command()
1640 	vi->ctrl->hdr.class = class;  in virtnet_send_command()
1641 	vi->ctrl->hdr.cmd = cmd;  in virtnet_send_command()
1643 	sg_init_one(&hdr, &vi->ctrl->hdr, sizeof(vi->ctrl->hdr));  in virtnet_send_command()
1650 	sg_init_one(&stat, &vi->ctrl->status, sizeof(vi->ctrl->status));  in virtnet_send_command()
1654 	virtqueue_add_sgs(vi->cvq, sgs, out_num, 1, vi, GFP_ATOMIC);  in virtnet_send_command()
1656 	if (unlikely(!virtqueue_kick(vi->cvq)))  in virtnet_send_command()
1657 		return vi->ctrl->status == VIRTIO_NET_OK;  in virtnet_send_command()
1662 	while (!virtqueue_get_buf(vi->cvq, &tmp) &&  in virtnet_send_command()
1663 	       !virtqueue_is_broken(vi->cvq))  in virtnet_send_command()
1666 	return vi->ctrl->status == VIRTIO_NET_OK;  in virtnet_send_command()
1671 	struct virtnet_info *vi = netdev_priv(dev);  in virtnet_set_mac_address()  local
1672 	struct virtio_device *vdev = vi->vdev;  in virtnet_set_mac_address()
1677 	if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_STANDBY))  in virtnet_set_mac_address()
1690 		if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_MAC,  in virtnet_set_mac_address()
1719 	struct virtnet_info *vi = netdev_priv(dev);  in virtnet_stats()  local
1723 	for (i = 0; i < vi->max_queue_pairs; i++) {  in virtnet_stats()
1725 		struct receive_queue *rq = &vi->rq[i];  in virtnet_stats()
1726 		struct send_queue *sq = &vi->sq[i];  in virtnet_stats()
1754 static void virtnet_ack_link_announce(struct virtnet_info *vi)  in virtnet_ack_link_announce()  argument
1757 	if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_ANNOUNCE,  in virtnet_ack_link_announce()
1759 		dev_warn(&vi->dev->dev, "Failed to ack link announce.\n");  in virtnet_ack_link_announce()
1763 static int _virtnet_set_queues(struct virtnet_info *vi, u16 queue_pairs)  in _virtnet_set_queues()  argument
1766 	struct net_device *dev = vi->dev;  in _virtnet_set_queues()
1768 	if (!vi->has_cvq || !virtio_has_feature(vi->vdev, VIRTIO_NET_F_MQ))  in _virtnet_set_queues()
1771 	vi->ctrl->mq.virtqueue_pairs = cpu_to_virtio16(vi->vdev, queue_pairs);  in _virtnet_set_queues()
1772 	sg_init_one(&sg, &vi->ctrl->mq, sizeof(vi->ctrl->mq));  in _virtnet_set_queues()
1774 	if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_MQ,  in _virtnet_set_queues()
1780 		vi->curr_queue_pairs = queue_pairs;  in _virtnet_set_queues()
1783 			schedule_delayed_work(&vi->refill, 0);  in _virtnet_set_queues()
1789 static int virtnet_set_queues(struct virtnet_info *vi, u16 queue_pairs)  in virtnet_set_queues()  argument
1794 	err = _virtnet_set_queues(vi, queue_pairs);  in virtnet_set_queues()
1801 	struct virtnet_info *vi = netdev_priv(dev);  in virtnet_close()  local
1805 	cancel_delayed_work_sync(&vi->refill);  in virtnet_close()
1807 	for (i = 0; i < vi->max_queue_pairs; i++) {  in virtnet_close()
1808 		xdp_rxq_info_unreg(&vi->rq[i].xdp_rxq);  in virtnet_close()
1809 		napi_disable(&vi->rq[i].napi);  in virtnet_close()
1810 		virtnet_napi_tx_disable(&vi->sq[i].napi);  in virtnet_close()
1818 	struct virtnet_info *vi = netdev_priv(dev);  in virtnet_set_rx_mode()  local
1828 	if (!virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_RX))  in virtnet_set_rx_mode()
1831 	vi->ctrl->promisc = ((dev->flags & IFF_PROMISC) != 0);  in virtnet_set_rx_mode()
1832 	vi->ctrl->allmulti = ((dev->flags & IFF_ALLMULTI) != 0);  in virtnet_set_rx_mode()
1834 	sg_init_one(sg, &vi->ctrl->promisc, sizeof(vi->ctrl->promisc));  in virtnet_set_rx_mode()
1836 	if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_RX,  in virtnet_set_rx_mode()
1839 			 vi->ctrl->promisc ? "en" : "dis");  in virtnet_set_rx_mode()
1841 	sg_init_one(sg, &vi->ctrl->allmulti, sizeof(vi->ctrl->allmulti));  in virtnet_set_rx_mode()
1843 	if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_RX,  in virtnet_set_rx_mode()
1846 			 vi->ctrl->allmulti ? "en" : "dis");  in virtnet_set_rx_mode()
1860 	mac_data->entries = cpu_to_virtio32(vi->vdev, uc_count);  in virtnet_set_rx_mode()
1871 	mac_data->entries = cpu_to_virtio32(vi->vdev, mc_count);  in virtnet_set_rx_mode()
1879 	if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_MAC,  in virtnet_set_rx_mode()
1889 	struct virtnet_info *vi = netdev_priv(dev);  in virtnet_vlan_rx_add_vid()  local
1892 	vi->ctrl->vid = cpu_to_virtio16(vi->vdev, vid);  in virtnet_vlan_rx_add_vid()
1893 	sg_init_one(&sg, &vi->ctrl->vid, sizeof(vi->ctrl->vid));  in virtnet_vlan_rx_add_vid()
1895 	if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_VLAN,  in virtnet_vlan_rx_add_vid()
1904 	struct virtnet_info *vi = netdev_priv(dev);  in virtnet_vlan_rx_kill_vid()  local
1907 	vi->ctrl->vid = cpu_to_virtio16(vi->vdev, vid);  in virtnet_vlan_rx_kill_vid()
1908 	sg_init_one(&sg, &vi->ctrl->vid, sizeof(vi->ctrl->vid));  in virtnet_vlan_rx_kill_vid()
1910 	if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_VLAN,  in virtnet_vlan_rx_kill_vid()
1916 static void virtnet_clean_affinity(struct virtnet_info *vi)  in virtnet_clean_affinity()  argument
1920 	if (vi->affinity_hint_set) {  in virtnet_clean_affinity()
1921 		for (i = 0; i < vi->max_queue_pairs; i++) {  in virtnet_clean_affinity()
1922 			virtqueue_set_affinity(vi->rq[i].vq, NULL);  in virtnet_clean_affinity()
1923 			virtqueue_set_affinity(vi->sq[i].vq, NULL);  in virtnet_clean_affinity()
1926 		vi->affinity_hint_set = false;  in virtnet_clean_affinity()
1930 static void virtnet_set_affinity(struct virtnet_info *vi)  in virtnet_set_affinity()  argument
1940 		virtnet_clean_affinity(vi);  in virtnet_set_affinity()
1945 	stride = max_t(int, num_cpu / vi->curr_queue_pairs, 1);  in virtnet_set_affinity()
1946 	stragglers = num_cpu >= vi->curr_queue_pairs ?  in virtnet_set_affinity()
1947 			num_cpu % vi->curr_queue_pairs :  in virtnet_set_affinity()
1951 	for (i = 0; i < vi->curr_queue_pairs; i++) {  in virtnet_set_affinity()
1959 		virtqueue_set_affinity(vi->rq[i].vq, mask);  in virtnet_set_affinity()
1960 		virtqueue_set_affinity(vi->sq[i].vq, mask);  in virtnet_set_affinity()
1961 		__netif_set_xps_queue(vi->dev, cpumask_bits(mask), i, false);  in virtnet_set_affinity()
1965 	vi->affinity_hint_set = true;  in virtnet_set_affinity()
1971 	struct virtnet_info *vi = hlist_entry_safe(node, struct virtnet_info,  in virtnet_cpu_online()  local
1973 	virtnet_set_affinity(vi);  in virtnet_cpu_online()
1979 	struct virtnet_info *vi = hlist_entry_safe(node, struct virtnet_info,  in virtnet_cpu_dead()  local
1981 	virtnet_set_affinity(vi);  in virtnet_cpu_dead()
1987 	struct virtnet_info *vi = hlist_entry_safe(node, struct virtnet_info,  in virtnet_cpu_down_prep()  local
1990 	virtnet_clean_affinity(vi);  in virtnet_cpu_down_prep()
1996 static int virtnet_cpu_notif_add(struct virtnet_info *vi)  in virtnet_cpu_notif_add()  argument
2000 	ret = cpuhp_state_add_instance_nocalls(virtionet_online, &vi->node);  in virtnet_cpu_notif_add()
2004 					       &vi->node_dead);  in virtnet_cpu_notif_add()
2007 	cpuhp_state_remove_instance_nocalls(virtionet_online, &vi->node);  in virtnet_cpu_notif_add()
2011 static void virtnet_cpu_notif_remove(struct virtnet_info *vi)  in virtnet_cpu_notif_remove()  argument
2013 	cpuhp_state_remove_instance_nocalls(virtionet_online, &vi->node);  in virtnet_cpu_notif_remove()
2015 					    &vi->node_dead);  in virtnet_cpu_notif_remove()
2021 	struct virtnet_info *vi = netdev_priv(dev);  in virtnet_get_ringparam()  local
2023 	ring->rx_max_pending = virtqueue_get_vring_size(vi->rq[0].vq);  in virtnet_get_ringparam()
2024 	ring->tx_max_pending = virtqueue_get_vring_size(vi->sq[0].vq);  in virtnet_get_ringparam()
2033 	struct virtnet_info *vi = netdev_priv(dev);  in virtnet_get_drvinfo()  local
2034 	struct virtio_device *vdev = vi->vdev;  in virtnet_get_drvinfo()
2046 	struct virtnet_info *vi = netdev_priv(dev);  in virtnet_set_channels()  local
2056 	if (queue_pairs > vi->max_queue_pairs || queue_pairs == 0)  in virtnet_set_channels()
2063 	if (vi->rq[0].xdp_prog)  in virtnet_set_channels()
2067 	err = _virtnet_set_queues(vi, queue_pairs);  in virtnet_set_channels()
2072 		virtnet_set_affinity(vi);  in virtnet_set_channels()
2081 	struct virtnet_info *vi = netdev_priv(dev);  in virtnet_get_strings()  local
2087 		for (i = 0; i < vi->curr_queue_pairs; i++) {  in virtnet_get_strings()
2095 		for (i = 0; i < vi->curr_queue_pairs; i++) {  in virtnet_get_strings()
2108 	struct virtnet_info *vi = netdev_priv(dev);  in virtnet_get_sset_count()  local
2112 		return vi->curr_queue_pairs * (VIRTNET_RQ_STATS_LEN +  in virtnet_get_sset_count()
2122 	struct virtnet_info *vi = netdev_priv(dev);  in virtnet_get_ethtool_stats()  local
2127 	for (i = 0; i < vi->curr_queue_pairs; i++) {  in virtnet_get_ethtool_stats()
2128 		struct receive_queue *rq = &vi->rq[i];  in virtnet_get_ethtool_stats()
2141 	for (i = 0; i < vi->curr_queue_pairs; i++) {  in virtnet_get_ethtool_stats()
2142 		struct send_queue *sq = &vi->sq[i];  in virtnet_get_ethtool_stats()
2159 	struct virtnet_info *vi = netdev_priv(dev);  in virtnet_get_channels()  local
2161 	channels->combined_count = vi->curr_queue_pairs;  in virtnet_get_channels()
2162 	channels->max_combined = vi->max_queue_pairs;  in virtnet_get_channels()
2198 	struct virtnet_info *vi = netdev_priv(dev);  in virtnet_set_link_ksettings()  local
2207 	vi->speed = speed;  in virtnet_set_link_ksettings()
2208 	vi->duplex = cmd->base.duplex;  in virtnet_set_link_ksettings()
2216 	struct virtnet_info *vi = netdev_priv(dev);  in virtnet_get_link_ksettings()  local
2218 	cmd->base.speed = vi->speed;  in virtnet_get_link_ksettings()
2219 	cmd->base.duplex = vi->duplex;  in virtnet_get_link_ksettings()
2232 	struct virtnet_info *vi = netdev_priv(dev);  in virtnet_set_coalesce()  local
2245 	if (napi_weight ^ vi->sq[0].napi.weight) {  in virtnet_set_coalesce()
2248 		for (i = 0; i < vi->max_queue_pairs; i++)  in virtnet_set_coalesce()
2249 			vi->sq[i].napi.weight = napi_weight;  in virtnet_set_coalesce()
2262 	struct virtnet_info *vi = netdev_priv(dev);  in virtnet_get_coalesce()  local
2266 	if (vi->sq[0].napi.weight)  in virtnet_get_coalesce()
2274 	struct virtnet_info *vi = netdev_priv(dev);  in virtnet_init_settings()  local
2276 	vi->speed = SPEED_UNKNOWN;  in virtnet_init_settings()
2277 	vi->duplex = DUPLEX_UNKNOWN;  in virtnet_init_settings()
2280 static void virtnet_update_settings(struct virtnet_info *vi)  in virtnet_update_settings()  argument
2285 	if (!virtio_has_feature(vi->vdev, VIRTIO_NET_F_SPEED_DUPLEX))  in virtnet_update_settings()
2288 	speed = virtio_cread32(vi->vdev, offsetof(struct virtio_net_config,  in virtnet_update_settings()
2291 		vi->speed = speed;  in virtnet_update_settings()
2292 	duplex = virtio_cread8(vi->vdev, offsetof(struct virtio_net_config,  in virtnet_update_settings()
2295 		vi->duplex = duplex;  in virtnet_update_settings()
2316 	struct virtnet_info *vi = vdev->priv;  in virtnet_freeze_down()  local
2320 	flush_work(&vi->config_work);  in virtnet_freeze_down()
2322 	netif_tx_lock_bh(vi->dev);  in virtnet_freeze_down()
2323 	netif_device_detach(vi->dev);  in virtnet_freeze_down()
2324 	netif_tx_unlock_bh(vi->dev);  in virtnet_freeze_down()
2325 	cancel_delayed_work_sync(&vi->refill);  in virtnet_freeze_down()
2327 	if (netif_running(vi->dev)) {  in virtnet_freeze_down()
2328 		for (i = 0; i < vi->max_queue_pairs; i++) {  in virtnet_freeze_down()
2329 			napi_disable(&vi->rq[i].napi);  in virtnet_freeze_down()
2330 			virtnet_napi_tx_disable(&vi->sq[i].napi);  in virtnet_freeze_down()
2335 static int init_vqs(struct virtnet_info *vi);
2339 	struct virtnet_info *vi = vdev->priv;  in virtnet_restore_up()  local
2342 	err = init_vqs(vi);  in virtnet_restore_up()
2348 	if (netif_running(vi->dev)) {  in virtnet_restore_up()
2349 		for (i = 0; i < vi->curr_queue_pairs; i++)  in virtnet_restore_up()
2350 			if (!try_fill_recv(vi, &vi->rq[i], GFP_KERNEL))  in virtnet_restore_up()
2351 				schedule_delayed_work(&vi->refill, 0);  in virtnet_restore_up()
2353 		for (i = 0; i < vi->max_queue_pairs; i++) {  in virtnet_restore_up()
2354 			virtnet_napi_enable(vi->rq[i].vq, &vi->rq[i].napi);  in virtnet_restore_up()
2355 			virtnet_napi_tx_enable(vi, vi->sq[i].vq,  in virtnet_restore_up()
2356 					       &vi->sq[i].napi);  in virtnet_restore_up()
2360 	netif_tx_lock_bh(vi->dev);  in virtnet_restore_up()
2361 	netif_device_attach(vi->dev);  in virtnet_restore_up()
2362 	netif_tx_unlock_bh(vi->dev);  in virtnet_restore_up()
2366 static int virtnet_set_guest_offloads(struct virtnet_info *vi, u64 offloads)  in virtnet_set_guest_offloads()  argument
2369 	vi->ctrl->offloads = cpu_to_virtio64(vi->vdev, offloads);  in virtnet_set_guest_offloads()
2371 	sg_init_one(&sg, &vi->ctrl->offloads, sizeof(vi->ctrl->offloads));  in virtnet_set_guest_offloads()
2373 	if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_GUEST_OFFLOADS,  in virtnet_set_guest_offloads()
2375 		dev_warn(&vi->dev->dev, "Fail to set guest offload.\n");  in virtnet_set_guest_offloads()
2382 static int virtnet_clear_guest_offloads(struct virtnet_info *vi)  in virtnet_clear_guest_offloads()  argument
2386 	if (!vi->guest_offloads)  in virtnet_clear_guest_offloads()
2389 	return virtnet_set_guest_offloads(vi, offloads);  in virtnet_clear_guest_offloads()
2392 static int virtnet_restore_guest_offloads(struct virtnet_info *vi)  in virtnet_restore_guest_offloads()  argument
2394 	u64 offloads = vi->guest_offloads;  in virtnet_restore_guest_offloads()
2396 	if (!vi->guest_offloads)  in virtnet_restore_guest_offloads()
2399 	return virtnet_set_guest_offloads(vi, offloads);  in virtnet_restore_guest_offloads()
2406 	struct virtnet_info *vi = netdev_priv(dev);  in virtnet_xdp_set()  local
2411 	if (!virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_GUEST_OFFLOADS)  in virtnet_xdp_set()
2412 	    && (virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_TSO4) ||  in virtnet_xdp_set()
2413 	        virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_TSO6) ||  in virtnet_xdp_set()
2414 	        virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_ECN) ||  in virtnet_xdp_set()
2415 		virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_UFO) ||  in virtnet_xdp_set()
2416 		virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_CSUM))) {  in virtnet_xdp_set()
2421 	if (vi->mergeable_rx_bufs && !vi->any_header_sg) {  in virtnet_xdp_set()
2432 	curr_qp = vi->curr_queue_pairs - vi->xdp_queue_pairs;  in virtnet_xdp_set()
2437 	if (curr_qp + xdp_qp > vi->max_queue_pairs) {  in virtnet_xdp_set()
2440 			    curr_qp + xdp_qp, vi->max_queue_pairs);  in virtnet_xdp_set()
2444 	old_prog = rtnl_dereference(vi->rq[0].xdp_prog);  in virtnet_xdp_set()
2449 		prog = bpf_prog_add(prog, vi->max_queue_pairs - 1);  in virtnet_xdp_set()
2456 		for (i = 0; i < vi->max_queue_pairs; i++) {  in virtnet_xdp_set()
2457 			napi_disable(&vi->rq[i].napi);  in virtnet_xdp_set()
2458 			virtnet_napi_tx_disable(&vi->sq[i].napi);  in virtnet_xdp_set()
2463 		for (i = 0; i < vi->max_queue_pairs; i++) {  in virtnet_xdp_set()
2464 			rcu_assign_pointer(vi->rq[i].xdp_prog, prog);  in virtnet_xdp_set()
2466 				virtnet_restore_guest_offloads(vi);  in virtnet_xdp_set()
2471 	err = _virtnet_set_queues(vi, curr_qp + xdp_qp);  in virtnet_xdp_set()
2475 	vi->xdp_queue_pairs = xdp_qp;  in virtnet_xdp_set()
2478 		for (i = 0; i < vi->max_queue_pairs; i++) {  in virtnet_xdp_set()
2479 			rcu_assign_pointer(vi->rq[i].xdp_prog, prog);  in virtnet_xdp_set()
2481 				virtnet_clear_guest_offloads(vi);  in virtnet_xdp_set()
2485 	for (i = 0; i < vi->max_queue_pairs; i++) {  in virtnet_xdp_set()
2489 			virtnet_napi_enable(vi->rq[i].vq, &vi->rq[i].napi);  in virtnet_xdp_set()
2490 			virtnet_napi_tx_enable(vi, vi->sq[i].vq,  in virtnet_xdp_set()
2491 					       &vi->sq[i].napi);  in virtnet_xdp_set()
2499 		virtnet_clear_guest_offloads(vi);  in virtnet_xdp_set()
2500 		for (i = 0; i < vi->max_queue_pairs; i++)  in virtnet_xdp_set()
2501 			rcu_assign_pointer(vi->rq[i].xdp_prog, old_prog);  in virtnet_xdp_set()
2505 		for (i = 0; i < vi->max_queue_pairs; i++) {  in virtnet_xdp_set()
2506 			virtnet_napi_enable(vi->rq[i].vq, &vi->rq[i].napi);  in virtnet_xdp_set()
2507 			virtnet_napi_tx_enable(vi, vi->sq[i].vq,  in virtnet_xdp_set()
2508 					       &vi->sq[i].napi);  in virtnet_xdp_set()
2512 		bpf_prog_sub(prog, vi->max_queue_pairs - 1);  in virtnet_xdp_set()
2518 	struct virtnet_info *vi = netdev_priv(dev);  in virtnet_xdp_query()  local
2522 	for (i = 0; i < vi->max_queue_pairs; i++) {  in virtnet_xdp_query()
2523 		xdp_prog = rtnl_dereference(vi->rq[i].xdp_prog);  in virtnet_xdp_query()
2546 	struct virtnet_info *vi = netdev_priv(dev);  in virtnet_get_phys_port_name()  local
2549 	if (!virtio_has_feature(vi->vdev, VIRTIO_NET_F_STANDBY))  in virtnet_get_phys_port_name()
2562 	struct virtnet_info *vi = netdev_priv(dev);  in virtnet_set_features()  local
2567 		if (vi->xdp_queue_pairs)  in virtnet_set_features()
2571 			offloads = vi->guest_offloads_capable;  in virtnet_set_features()
2575 		err = virtnet_set_guest_offloads(vi, offloads);  in virtnet_set_features()
2578 		vi->guest_offloads = offloads;  in virtnet_set_features()
2603 	struct virtnet_info *vi =  in virtnet_config_changed_work()  local
2607 	if (virtio_cread_feature(vi->vdev, VIRTIO_NET_F_STATUS,  in virtnet_config_changed_work()
2612 		netdev_notify_peers(vi->dev);  in virtnet_config_changed_work()
2613 		virtnet_ack_link_announce(vi);  in virtnet_config_changed_work()
2619 	if (vi->status == v)  in virtnet_config_changed_work()
2622 	vi->status = v;  in virtnet_config_changed_work()
2624 	if (vi->status & VIRTIO_NET_S_LINK_UP) {  in virtnet_config_changed_work()
2625 		virtnet_update_settings(vi);  in virtnet_config_changed_work()
2626 		netif_carrier_on(vi->dev);  in virtnet_config_changed_work()
2627 		netif_tx_wake_all_queues(vi->dev);  in virtnet_config_changed_work()
2629 		netif_carrier_off(vi->dev);  in virtnet_config_changed_work()
2630 		netif_tx_stop_all_queues(vi->dev);  in virtnet_config_changed_work()
2636 	struct virtnet_info *vi = vdev->priv;  in virtnet_config_changed()  local
2638 	schedule_work(&vi->config_work);  in virtnet_config_changed()
2641 static void virtnet_free_queues(struct virtnet_info *vi)  in virtnet_free_queues()  argument
2645 	for (i = 0; i < vi->max_queue_pairs; i++) {  in virtnet_free_queues()
2646 		napi_hash_del(&vi->rq[i].napi);  in virtnet_free_queues()
2647 		netif_napi_del(&vi->rq[i].napi);  in virtnet_free_queues()
2648 		netif_napi_del(&vi->sq[i].napi);  in virtnet_free_queues()
2656 	kfree(vi->rq);  in virtnet_free_queues()
2657 	kfree(vi->sq);  in virtnet_free_queues()
2658 	kfree(vi->ctrl);  in virtnet_free_queues()
2661 static void _free_receive_bufs(struct virtnet_info *vi)  in _free_receive_bufs()  argument
2666 	for (i = 0; i < vi->max_queue_pairs; i++) {  in _free_receive_bufs()
2667 		while (vi->rq[i].pages)  in _free_receive_bufs()
2668 			__free_pages(get_a_page(&vi->rq[i], GFP_KERNEL), 0);  in _free_receive_bufs()
2670 		old_prog = rtnl_dereference(vi->rq[i].xdp_prog);  in _free_receive_bufs()
2671 		RCU_INIT_POINTER(vi->rq[i].xdp_prog, NULL);  in _free_receive_bufs()
2677 static void free_receive_bufs(struct virtnet_info *vi)  in free_receive_bufs()  argument
2680 	_free_receive_bufs(vi);  in free_receive_bufs()
2684 static void free_receive_page_frags(struct virtnet_info *vi)  in free_receive_page_frags()  argument
2687 	for (i = 0; i < vi->max_queue_pairs; i++)  in free_receive_page_frags()
2688 		if (vi->rq[i].alloc_frag.page)  in free_receive_page_frags()
2689 			put_page(vi->rq[i].alloc_frag.page);  in free_receive_page_frags()
2692 static void free_unused_bufs(struct virtnet_info *vi)  in free_unused_bufs()  argument
2697 	for (i = 0; i < vi->max_queue_pairs; i++) {  in free_unused_bufs()
2698 		struct virtqueue *vq = vi->sq[i].vq;  in free_unused_bufs()
2707 	for (i = 0; i < vi->max_queue_pairs; i++) {  in free_unused_bufs()
2708 		struct virtqueue *vq = vi->rq[i].vq;  in free_unused_bufs()
2711 			if (vi->mergeable_rx_bufs) {  in free_unused_bufs()
2713 			} else if (vi->big_packets) {  in free_unused_bufs()
2714 				give_pages(&vi->rq[i], buf);  in free_unused_bufs()
2722 static void virtnet_del_vqs(struct virtnet_info *vi)  in virtnet_del_vqs()  argument
2724 	struct virtio_device *vdev = vi->vdev;  in virtnet_del_vqs()
2726 	virtnet_clean_affinity(vi);  in virtnet_del_vqs()
2730 	virtnet_free_queues(vi);  in virtnet_del_vqs()
2737 static unsigned int mergeable_min_buf_len(struct virtnet_info *vi, struct virtqueue *vq)  in mergeable_min_buf_len()  argument
2741 	unsigned int packet_len = vi->big_packets ? IP_MAX_MTU : vi->dev->max_mtu;  in mergeable_min_buf_len()
2749 static int virtnet_find_vqs(struct virtnet_info *vi)  in virtnet_find_vqs()  argument
2762 	total_vqs = vi->max_queue_pairs * 2 +  in virtnet_find_vqs()
2763 		    virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_VQ);  in virtnet_find_vqs()
2775 	if (!vi->big_packets || vi->mergeable_rx_bufs) {  in virtnet_find_vqs()
2784 	if (vi->has_cvq) {  in virtnet_find_vqs()
2790 	for (i = 0; i < vi->max_queue_pairs; i++) {  in virtnet_find_vqs()
2793 		sprintf(vi->rq[i].name, "input.%d", i);  in virtnet_find_vqs()
2794 		sprintf(vi->sq[i].name, "output.%d", i);  in virtnet_find_vqs()
2795 		names[rxq2vq(i)] = vi->rq[i].name;  in virtnet_find_vqs()
2796 		names[txq2vq(i)] = vi->sq[i].name;  in virtnet_find_vqs()
2801 	ret = vi->vdev->config->find_vqs(vi->vdev, total_vqs, vqs, callbacks,  in virtnet_find_vqs()
2806 	if (vi->has_cvq) {  in virtnet_find_vqs()
2807 		vi->cvq = vqs[total_vqs - 1];  in virtnet_find_vqs()
2808 		if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_VLAN))  in virtnet_find_vqs()
2809 			vi->dev->features |= NETIF_F_HW_VLAN_CTAG_FILTER;  in virtnet_find_vqs()
2812 	for (i = 0; i < vi->max_queue_pairs; i++) {  in virtnet_find_vqs()
2813 		vi->rq[i].vq = vqs[rxq2vq(i)];  in virtnet_find_vqs()
2814 		vi->rq[i].min_buf_len = mergeable_min_buf_len(vi, vi->rq[i].vq);  in virtnet_find_vqs()
2815 		vi->sq[i].vq = vqs[txq2vq(i)];  in virtnet_find_vqs()
2833 static int virtnet_alloc_queues(struct virtnet_info *vi)  in virtnet_alloc_queues()  argument
2837 	vi->ctrl = kzalloc(sizeof(*vi->ctrl), GFP_KERNEL);  in virtnet_alloc_queues()
2838 	if (!vi->ctrl)  in virtnet_alloc_queues()
2840 	vi->sq = kcalloc(vi->max_queue_pairs, sizeof(*vi->sq), GFP_KERNEL);  in virtnet_alloc_queues()
2841 	if (!vi->sq)  in virtnet_alloc_queues()
2843 	vi->rq = kcalloc(vi->max_queue_pairs, sizeof(*vi->rq), GFP_KERNEL);  in virtnet_alloc_queues()
2844 	if (!vi->rq)  in virtnet_alloc_queues()
2847 	INIT_DELAYED_WORK(&vi->refill, refill_work);  in virtnet_alloc_queues()
2848 	for (i = 0; i < vi->max_queue_pairs; i++) {  in virtnet_alloc_queues()
2849 		vi->rq[i].pages = NULL;  in virtnet_alloc_queues()
2850 		netif_napi_add(vi->dev, &vi->rq[i].napi, virtnet_poll,  in virtnet_alloc_queues()
2852 		netif_tx_napi_add(vi->dev, &vi->sq[i].napi, virtnet_poll_tx,  in virtnet_alloc_queues()
2855 		sg_init_table(vi->rq[i].sg, ARRAY_SIZE(vi->rq[i].sg));  in virtnet_alloc_queues()
2856 		ewma_pkt_len_init(&vi->rq[i].mrg_avg_pkt_len);  in virtnet_alloc_queues()
2857 		sg_init_table(vi->sq[i].sg, ARRAY_SIZE(vi->sq[i].sg));  in virtnet_alloc_queues()
2859 		u64_stats_init(&vi->rq[i].stats.syncp);  in virtnet_alloc_queues()
2860 		u64_stats_init(&vi->sq[i].stats.syncp);  in virtnet_alloc_queues()
2866 	kfree(vi->sq);  in virtnet_alloc_queues()
2868 	kfree(vi->ctrl);  in virtnet_alloc_queues()
2873 static int init_vqs(struct virtnet_info *vi)  in init_vqs()  argument
2878 	ret = virtnet_alloc_queues(vi);  in init_vqs()
2882 	ret = virtnet_find_vqs(vi);  in init_vqs()
2887 	virtnet_set_affinity(vi);  in init_vqs()
2893 	virtnet_free_queues(vi);  in init_vqs()
2902 	struct virtnet_info *vi = netdev_priv(queue->dev);  in mergeable_rx_buffer_size_show()  local
2904 	unsigned int headroom = virtnet_get_headroom(vi);  in mergeable_rx_buffer_size_show()
2908 	BUG_ON(queue_index >= vi->max_queue_pairs);  in mergeable_rx_buffer_size_show()
2909 	avg = &vi->rq[queue_index].mrg_avg_pkt_len;  in mergeable_rx_buffer_size_show()
2911 		       get_mergeable_buf_len(&vi->rq[queue_index], avg,  in mergeable_rx_buffer_size_show()
2992 	struct virtnet_info *vi;  in virtnet_probe()  local
3068 	vi = netdev_priv(dev);  in virtnet_probe()
3069 	vi->dev = dev;  in virtnet_probe()
3070 	vi->vdev = vdev;  in virtnet_probe()
3071 	vdev->priv = vi;  in virtnet_probe()
3073 	INIT_WORK(&vi->config_work, virtnet_config_changed_work);  in virtnet_probe()
3080 		vi->big_packets = true;  in virtnet_probe()
3083 		vi->mergeable_rx_bufs = true;  in virtnet_probe()
3087 		vi->hdr_len = sizeof(struct virtio_net_hdr_mrg_rxbuf);  in virtnet_probe()
3089 		vi->hdr_len = sizeof(struct virtio_net_hdr);  in virtnet_probe()
3093 		vi->any_header_sg = true;  in virtnet_probe()
3096 		vi->has_cvq = true;  in virtnet_probe()
3117 			vi->big_packets = true;  in virtnet_probe()
3120 	if (vi->any_header_sg)  in virtnet_probe()
3121 		dev->needed_headroom = vi->hdr_len;  in virtnet_probe()
3125 		vi->curr_queue_pairs = max_queue_pairs;  in virtnet_probe()
3127 		vi->curr_queue_pairs = num_online_cpus();  in virtnet_probe()
3128 	vi->max_queue_pairs = max_queue_pairs;  in virtnet_probe()
3131 	err = init_vqs(vi);  in virtnet_probe()
3136 	if (vi->mergeable_rx_bufs)  in virtnet_probe()
3139 	netif_set_real_num_tx_queues(dev, vi->curr_queue_pairs);  in virtnet_probe()
3140 	netif_set_real_num_rx_queues(dev, vi->curr_queue_pairs);  in virtnet_probe()
3145 		vi->failover = net_failover_create(vi->dev);  in virtnet_probe()
3146 		if (IS_ERR(vi->failover)) {  in virtnet_probe()
3147 			err = PTR_ERR(vi->failover);  in virtnet_probe()
3160 	err = virtnet_cpu_notif_add(vi);  in virtnet_probe()
3166 	virtnet_set_queues(vi, vi->curr_queue_pairs);  in virtnet_probe()
3171 	if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_STATUS)) {  in virtnet_probe()
3172 		schedule_work(&vi->config_work);  in virtnet_probe()
3174 		vi->status = VIRTIO_NET_S_LINK_UP;  in virtnet_probe()
3175 		virtnet_update_settings(vi);  in virtnet_probe()
3180 		if (virtio_has_feature(vi->vdev, guest_offloads[i]))  in virtnet_probe()
3181 			set_bit(guest_offloads[i], &vi->guest_offloads);  in virtnet_probe()
3182 	vi->guest_offloads_capable = vi->guest_offloads;  in virtnet_probe()
3190 	vi->vdev->config->reset(vdev);  in virtnet_probe()
3194 	net_failover_destroy(vi->failover);  in virtnet_probe()
3196 	cancel_delayed_work_sync(&vi->refill);  in virtnet_probe()
3197 	free_receive_page_frags(vi);  in virtnet_probe()
3198 	virtnet_del_vqs(vi);  in virtnet_probe()
3204 static void remove_vq_common(struct virtnet_info *vi)  in remove_vq_common()  argument
3206 	vi->vdev->config->reset(vi->vdev);  in remove_vq_common()
3209 	free_unused_bufs(vi);  in remove_vq_common()
3211 	free_receive_bufs(vi);  in remove_vq_common()
3213 	free_receive_page_frags(vi);  in remove_vq_common()
3215 	virtnet_del_vqs(vi);  in remove_vq_common()
3220 	struct virtnet_info *vi = vdev->priv;  in virtnet_remove()  local
3222 	virtnet_cpu_notif_remove(vi);  in virtnet_remove()
3225 	flush_work(&vi->config_work);  in virtnet_remove()
3227 	unregister_netdev(vi->dev);  in virtnet_remove()
3229 	net_failover_destroy(vi->failover);  in virtnet_remove()
3231 	remove_vq_common(vi);  in virtnet_remove()
3233 	free_netdev(vi->dev);  in virtnet_remove()
3238 	struct virtnet_info *vi = vdev->priv;  in virtnet_freeze()  local
3240 	virtnet_cpu_notif_remove(vi);  in virtnet_freeze()
3242 	remove_vq_common(vi);  in virtnet_freeze()
3249 	struct virtnet_info *vi = vdev->priv;  in virtnet_restore()  local
3255 	virtnet_set_queues(vi, vi->curr_queue_pairs);  in virtnet_restore()
3257 	err = virtnet_cpu_notif_add(vi);  in virtnet_restore()