176 static int vhost_net_buf_produce(struct vhost_net_virtqueue *nvq)  in vhost_net_buf_produce()  argument
178 	struct vhost_net_buf *rxq = &nvq->rxq;  in vhost_net_buf_produce()
181 	rxq->tail = ptr_ring_consume_batched(nvq->rx_ring, rxq->queue,  in vhost_net_buf_produce()
186 static void vhost_net_buf_unproduce(struct vhost_net_virtqueue *nvq)  in vhost_net_buf_unproduce()  argument
188 	struct vhost_net_buf *rxq = &nvq->rxq;  in vhost_net_buf_unproduce()
190 	if (nvq->rx_ring && !vhost_net_buf_is_empty(rxq)) {  in vhost_net_buf_unproduce()
191 		ptr_ring_unconsume(nvq->rx_ring, rxq->queue + rxq->head,  in vhost_net_buf_unproduce()
209 static int vhost_net_buf_peek(struct vhost_net_virtqueue *nvq)  in vhost_net_buf_peek()  argument
211 	struct vhost_net_buf *rxq = &nvq->rxq;  in vhost_net_buf_peek()
216 	if (!vhost_net_buf_produce(nvq))  in vhost_net_buf_peek()
361 	struct vhost_net_virtqueue *nvq =  in vhost_zerocopy_signal_used()  local
366 	for (i = nvq->done_idx; i != nvq->upend_idx; i = (i + 1) % UIO_MAXIOV) {  in vhost_zerocopy_signal_used()
376 		add = min(UIO_MAXIOV - nvq->done_idx, j);  in vhost_zerocopy_signal_used()
378 					    &vq->heads[nvq->done_idx], add);  in vhost_zerocopy_signal_used()
379 		nvq->done_idx = (nvq->done_idx + add) % UIO_MAXIOV;  in vhost_zerocopy_signal_used()
424 	struct vhost_net_virtqueue *nvq =  in vhost_net_disable_vq()  local
426 	struct vhost_poll *poll = n->poll + (nvq - n->vqs);  in vhost_net_disable_vq()
435 	struct vhost_net_virtqueue *nvq =  in vhost_net_enable_vq()  local
437 	struct vhost_poll *poll = n->poll + (nvq - n->vqs);  in vhost_net_enable_vq()
447 static void vhost_net_signal_used(struct vhost_net_virtqueue *nvq)  in vhost_net_signal_used()  argument
449 	struct vhost_virtqueue *vq = &nvq->vq;  in vhost_net_signal_used()
452 	if (!nvq->done_idx)  in vhost_net_signal_used()
455 	vhost_add_used_and_signal_n(dev, vq, vq->heads, nvq->done_idx);  in vhost_net_signal_used()
456 	nvq->done_idx = 0;  in vhost_net_signal_used()
460 			   struct vhost_net_virtqueue *nvq,  in vhost_tx_batch()  argument
466 		.num = nvq->batched_xdp,  in vhost_tx_batch()
467 		.ptr = nvq->xdp,  in vhost_tx_batch()
471 	if (nvq->batched_xdp == 0)  in vhost_tx_batch()
477 		vq_err(&nvq->vq, "Fail to batch sending packets\n");  in vhost_tx_batch()
482 	vhost_net_signal_used(nvq);  in vhost_tx_batch()
483 	nvq->batched_xdp = 0;  in vhost_tx_batch()
587 	struct vhost_net_virtqueue *nvq = &net->vqs[VHOST_NET_VQ_TX];  in vhost_exceeds_maxpend()  local
588 	struct vhost_virtqueue *vq = &nvq->vq;  in vhost_exceeds_maxpend()
590 	return (nvq->upend_idx + UIO_MAXIOV - nvq->done_idx) % UIO_MAXIOV >  in vhost_exceeds_maxpend()
607 		       struct vhost_net_virtqueue *nvq,  in get_tx_bufs()  argument
612 	struct vhost_virtqueue *vq = &nvq->vq;  in get_tx_bufs()
615 	ret = vhost_net_tx_get_vq_desc(net, nvq, out, in, msg, busyloop_intr);  in get_tx_bufs()
627 	*len = init_iov_iter(vq, &msg->msg_iter, nvq->vhost_hlen, *out);  in get_tx_bufs()
630 			*len, nvq->vhost_hlen);  in get_tx_bufs()
682 static int vhost_net_build_xdp(struct vhost_net_virtqueue *nvq,  in vhost_net_build_xdp()  argument
685 	struct vhost_virtqueue *vq = &nvq->vq;  in vhost_net_build_xdp()
691 	struct xdp_buff *xdp = &nvq->xdp[nvq->batched_xdp];  in vhost_net_build_xdp()
696 	int pad = SKB_DATA_ALIGN(VHOST_NET_RX_PAD + headroom + nvq->sock_hlen);  in vhost_net_build_xdp()
697 	int sock_hlen = nvq->sock_hlen;  in vhost_net_build_xdp()
701 	if (unlikely(len < nvq->sock_hlen))  in vhost_net_build_xdp()
752 	++nvq->batched_xdp;  in vhost_net_build_xdp()
759 	struct vhost_net_virtqueue *nvq = &net->vqs[VHOST_NET_VQ_TX];  in handle_tx_copy()  local
760 	struct vhost_virtqueue *vq = &nvq->vq;  in handle_tx_copy()
778 		if (nvq->done_idx == VHOST_NET_BATCH)  in handle_tx_copy()
779 			vhost_tx_batch(net, nvq, sock, &msg);  in handle_tx_copy()
781 		head = get_tx_bufs(net, nvq, &msg, &out, &in, &len,  in handle_tx_copy()
804 			err = vhost_net_build_xdp(nvq, &msg.msg_iter);  in handle_tx_copy()
808 				vhost_tx_batch(net, nvq, sock, &msg);  in handle_tx_copy()
818 			vhost_tx_batch(net, nvq, sock, &msg);  in handle_tx_copy()
838 		vq->heads[nvq->done_idx].id = cpu_to_vhost32(vq, head);  in handle_tx_copy()
839 		vq->heads[nvq->done_idx].len = 0;  in handle_tx_copy()
840 		++nvq->done_idx;  in handle_tx_copy()
843 	vhost_tx_batch(net, nvq, sock, &msg);  in handle_tx_copy()
848 	struct vhost_net_virtqueue *nvq = &net->vqs[VHOST_NET_VQ_TX];  in handle_tx_zerocopy()  local
849 	struct vhost_virtqueue *vq = &nvq->vq;  in handle_tx_zerocopy()
873 		head = get_tx_bufs(net, nvq, &msg, &out, &in, &len,  in handle_tx_zerocopy()
896 			ubuf = nvq->ubuf_info + nvq->upend_idx;  in handle_tx_zerocopy()
898 			vq->heads[nvq->upend_idx].id = cpu_to_vhost32(vq, head);  in handle_tx_zerocopy()
899 			vq->heads[nvq->upend_idx].len = VHOST_DMA_IN_PROGRESS;  in handle_tx_zerocopy()
901 			ubuf->ctx = nvq->ubufs;  in handle_tx_zerocopy()
902 			ubuf->desc = nvq->upend_idx;  in handle_tx_zerocopy()
908 			ubufs = nvq->ubufs;  in handle_tx_zerocopy()
910 			nvq->upend_idx = (nvq->upend_idx + 1) % UIO_MAXIOV;  in handle_tx_zerocopy()
928 				nvq->upend_idx = ((unsigned)nvq->upend_idx - 1)  in handle_tx_zerocopy()
950 	struct vhost_net_virtqueue *nvq = &net->vqs[VHOST_NET_VQ_TX];  in handle_tx()  local
951 	struct vhost_virtqueue *vq = &nvq->vq;  in handle_tx()
1097 	struct vhost_net_virtqueue *nvq = &net->vqs[VHOST_NET_VQ_RX];  in handle_rx()  local
1098 	struct vhost_virtqueue *vq = &nvq->vq;  in handle_rx()
1134 	vhost_hlen = nvq->vhost_hlen;  in handle_rx()
1135 	sock_hlen = nvq->sock_hlen;  in handle_rx()
1148 		headcount = get_rx_bufs(vq, vq->heads + nvq->done_idx,  in handle_rx()
1169 		if (nvq->rx_ring)  in handle_rx()
1170 			msg.msg_control = vhost_net_buf_consume(&nvq->rxq);  in handle_rx()
1223 		nvq->done_idx += headcount;  in handle_rx()
1224 		if (nvq->done_idx > VHOST_NET_BATCH)  in handle_rx()
1225 			vhost_net_signal_used(nvq);  in handle_rx()
1237 	vhost_net_signal_used(nvq);  in handle_rx()
1343 	struct vhost_net_virtqueue *nvq =  in vhost_net_stop_vq()  local
1350 	vhost_net_buf_unproduce(nvq);  in vhost_net_stop_vq()
1351 	nvq->rx_ring = NULL;  in vhost_net_stop_vq()
1502 	struct vhost_net_virtqueue *nvq;  in vhost_net_set_backend()  local
1516 	nvq = &n->vqs[index];  in vhost_net_set_backend()
1542 		vhost_net_buf_unproduce(nvq);  in vhost_net_set_backend()
1550 			nvq->rx_ring = get_tap_ptr_ring(fd);  in vhost_net_set_backend()
1552 		oldubufs = nvq->ubufs;  in vhost_net_set_backend()
1553 		nvq->ubufs = ubufs;  in vhost_net_set_backend()