169 static int vhost_net_buf_produce(struct vhost_net_virtqueue *nvq)  in vhost_net_buf_produce()  argument
171 	struct vhost_net_buf *rxq = &nvq->rxq;  in vhost_net_buf_produce()
174 	rxq->tail = ptr_ring_consume_batched(nvq->rx_ring, rxq->queue,  in vhost_net_buf_produce()
179 static void vhost_net_buf_unproduce(struct vhost_net_virtqueue *nvq)  in vhost_net_buf_unproduce()  argument
181 	struct vhost_net_buf *rxq = &nvq->rxq;  in vhost_net_buf_unproduce()
183 	if (nvq->rx_ring && !vhost_net_buf_is_empty(rxq)) {  in vhost_net_buf_unproduce()
184 		ptr_ring_unconsume(nvq->rx_ring, rxq->queue + rxq->head,  in vhost_net_buf_unproduce()
202 static int vhost_net_buf_peek(struct vhost_net_virtqueue *nvq)  in vhost_net_buf_peek()  argument
204 	struct vhost_net_buf *rxq = &nvq->rxq;  in vhost_net_buf_peek()
209 	if (!vhost_net_buf_produce(nvq))  in vhost_net_buf_peek()
349 	struct vhost_net_virtqueue *nvq =  in vhost_zerocopy_signal_used()  local
354 	for (i = nvq->done_idx; i != nvq->upend_idx; i = (i + 1) % UIO_MAXIOV) {  in vhost_zerocopy_signal_used()
364 		add = min(UIO_MAXIOV - nvq->done_idx, j);  in vhost_zerocopy_signal_used()
366 					    &vq->heads[nvq->done_idx], add);  in vhost_zerocopy_signal_used()
367 		nvq->done_idx = (nvq->done_idx + add) % UIO_MAXIOV;  in vhost_zerocopy_signal_used()
412 	struct vhost_net_virtqueue *nvq =  in vhost_net_disable_vq()  local
414 	struct vhost_poll *poll = n->poll + (nvq - n->vqs);  in vhost_net_disable_vq()
423 	struct vhost_net_virtqueue *nvq =  in vhost_net_enable_vq()  local
425 	struct vhost_poll *poll = n->poll + (nvq - n->vqs);  in vhost_net_enable_vq()
435 static void vhost_net_signal_used(struct vhost_net_virtqueue *nvq)  in vhost_net_signal_used()  argument
437 	struct vhost_virtqueue *vq = &nvq->vq;  in vhost_net_signal_used()
440 	if (!nvq->done_idx)  in vhost_net_signal_used()
443 	vhost_add_used_and_signal_n(dev, vq, vq->heads, nvq->done_idx);  in vhost_net_signal_used()
444 	nvq->done_idx = 0;  in vhost_net_signal_used()
448 				    struct vhost_net_virtqueue *nvq,  in vhost_net_tx_get_vq_desc()  argument
452 	struct vhost_virtqueue *vq = &nvq->vq;  in vhost_net_tx_get_vq_desc()
459 			vhost_net_signal_used(nvq);  in vhost_net_tx_get_vq_desc()
481 	struct vhost_net_virtqueue *nvq = &net->vqs[VHOST_NET_VQ_TX];  in vhost_exceeds_maxpend()  local
482 	struct vhost_virtqueue *vq = &nvq->vq;  in vhost_exceeds_maxpend()
484 	return (nvq->upend_idx + UIO_MAXIOV - nvq->done_idx) % UIO_MAXIOV >  in vhost_exceeds_maxpend()
507 		       struct vhost_net_virtqueue *nvq,  in get_tx_bufs()  argument
512 	struct vhost_virtqueue *vq = &nvq->vq;  in get_tx_bufs()
515 	ret = vhost_net_tx_get_vq_desc(net, nvq, out, in, busyloop_intr);  in get_tx_bufs()
527 	*len = init_iov_iter(vq, &msg->msg_iter, nvq->vhost_hlen, *out);  in get_tx_bufs()
530 			*len, nvq->vhost_hlen);  in get_tx_bufs()
545 	struct vhost_net_virtqueue *nvq = &net->vqs[VHOST_NET_VQ_TX];  in handle_tx_copy()  local
546 	struct vhost_virtqueue *vq = &nvq->vq;  in handle_tx_copy()
563 		head = get_tx_bufs(net, nvq, &msg, &out, &in, &len,  in handle_tx_copy()
580 		vq->heads[nvq->done_idx].id = cpu_to_vhost32(vq, head);  in handle_tx_copy()
581 		vq->heads[nvq->done_idx].len = 0;  in handle_tx_copy()
599 		if (++nvq->done_idx >= VHOST_NET_BATCH)  in handle_tx_copy()
600 			vhost_net_signal_used(nvq);  in handle_tx_copy()
607 	vhost_net_signal_used(nvq);  in handle_tx_copy()
612 	struct vhost_net_virtqueue *nvq = &net->vqs[VHOST_NET_VQ_TX];  in handle_tx_zerocopy()  local
613 	struct vhost_virtqueue *vq = &nvq->vq;  in handle_tx_zerocopy()
636 		head = get_tx_bufs(net, nvq, &msg, &out, &in, &len,  in handle_tx_zerocopy()
659 			ubuf = nvq->ubuf_info + nvq->upend_idx;  in handle_tx_zerocopy()
661 			vq->heads[nvq->upend_idx].id = cpu_to_vhost32(vq, head);  in handle_tx_zerocopy()
662 			vq->heads[nvq->upend_idx].len = VHOST_DMA_IN_PROGRESS;  in handle_tx_zerocopy()
664 			ubuf->ctx = nvq->ubufs;  in handle_tx_zerocopy()
665 			ubuf->desc = nvq->upend_idx;  in handle_tx_zerocopy()
669 			ubufs = nvq->ubufs;  in handle_tx_zerocopy()
671 			nvq->upend_idx = (nvq->upend_idx + 1) % UIO_MAXIOV;  in handle_tx_zerocopy()
689 				nvq->upend_idx = ((unsigned)nvq->upend_idx - 1)  in handle_tx_zerocopy()
715 	struct vhost_net_virtqueue *nvq = &net->vqs[VHOST_NET_VQ_TX];  in handle_tx()  local
716 	struct vhost_virtqueue *vq = &nvq->vq;  in handle_tx()
900 	struct vhost_net_virtqueue *nvq = &net->vqs[VHOST_NET_VQ_RX];  in handle_rx()  local
901 	struct vhost_virtqueue *vq = &nvq->vq;  in handle_rx()
937 	vhost_hlen = nvq->vhost_hlen;  in handle_rx()
938 	sock_hlen = nvq->sock_hlen;  in handle_rx()
948 		headcount = get_rx_bufs(vq, vq->heads + nvq->done_idx,  in handle_rx()
969 		if (nvq->rx_ring)  in handle_rx()
970 			msg.msg_control = vhost_net_buf_consume(&nvq->rxq);  in handle_rx()
1023 		nvq->done_idx += headcount;  in handle_rx()
1024 		if (nvq->done_idx > VHOST_NET_BATCH)  in handle_rx()
1025 			vhost_net_signal_used(nvq);  in handle_rx()
1039 	vhost_net_signal_used(nvq);  in handle_rx()
1130 	struct vhost_net_virtqueue *nvq =  in vhost_net_stop_vq()  local
1137 	vhost_net_buf_unproduce(nvq);  in vhost_net_stop_vq()
1138 	nvq->rx_ring = NULL;  in vhost_net_stop_vq()
1286 	struct vhost_net_virtqueue *nvq;  in vhost_net_set_backend()  local
1300 	nvq = &n->vqs[index];  in vhost_net_set_backend()
1326 		vhost_net_buf_unproduce(nvq);  in vhost_net_set_backend()
1334 			nvq->rx_ring = get_tap_ptr_ring(fd);  in vhost_net_set_backend()
1336 		oldubufs = nvq->ubufs;  in vhost_net_set_backend()
1337 		nvq->ubufs = ubufs;  in vhost_net_set_backend()