Lines Matching refs:vq
98 struct vhost_virtqueue *vq; member
109 struct vhost_virtqueue vq; member
221 static void vhost_net_enable_zcopy(int vq) in vhost_net_enable_zcopy() argument
223 vhost_net_zcopy_mask |= 0x1 << vq; in vhost_net_enable_zcopy()
227 vhost_net_ubuf_alloc(struct vhost_virtqueue *vq, bool zcopy) in vhost_net_ubuf_alloc() argument
238 ubufs->vq = vq; in vhost_net_ubuf_alloc()
347 struct vhost_virtqueue *vq) in vhost_zerocopy_signal_used() argument
350 container_of(vq, struct vhost_net_virtqueue, vq); in vhost_zerocopy_signal_used()
355 if (vq->heads[i].len == VHOST_DMA_FAILED_LEN) in vhost_zerocopy_signal_used()
357 if (VHOST_DMA_IS_DONE(vq->heads[i].len)) { in vhost_zerocopy_signal_used()
358 vq->heads[i].len = VHOST_DMA_CLEAR_LEN; in vhost_zerocopy_signal_used()
365 vhost_add_used_and_signal_n(vq->dev, vq, in vhost_zerocopy_signal_used()
366 &vq->heads[nvq->done_idx], add); in vhost_zerocopy_signal_used()
375 struct vhost_virtqueue *vq = ubufs->vq; in vhost_zerocopy_callback() local
381 vq->heads[ubuf->desc].len = success ? in vhost_zerocopy_callback()
393 vhost_poll_queue(&vq->poll); in vhost_zerocopy_callback()
410 struct vhost_virtqueue *vq) in vhost_net_disable_vq() argument
413 container_of(vq, struct vhost_net_virtqueue, vq); in vhost_net_disable_vq()
415 if (!vq->private_data) in vhost_net_disable_vq()
421 struct vhost_virtqueue *vq) in vhost_net_enable_vq() argument
424 container_of(vq, struct vhost_net_virtqueue, vq); in vhost_net_enable_vq()
428 sock = vq->private_data; in vhost_net_enable_vq()
437 struct vhost_virtqueue *vq = &nvq->vq; in vhost_net_signal_used() local
438 struct vhost_dev *dev = vq->dev; in vhost_net_signal_used()
443 vhost_add_used_and_signal_n(dev, vq, vq->heads, nvq->done_idx); in vhost_net_signal_used()
452 struct vhost_virtqueue *vq = &nvq->vq; in vhost_net_tx_get_vq_desc() local
454 int r = vhost_get_vq_desc(vq, vq->iov, ARRAY_SIZE(vq->iov), in vhost_net_tx_get_vq_desc()
457 if (r == vq->num && vq->busyloop_timeout) { in vhost_net_tx_get_vq_desc()
458 if (!vhost_sock_zcopy(vq->private_data)) in vhost_net_tx_get_vq_desc()
461 endtime = busy_clock() + vq->busyloop_timeout; in vhost_net_tx_get_vq_desc()
463 if (vhost_has_work(vq->dev)) { in vhost_net_tx_get_vq_desc()
467 if (!vhost_vq_avail_empty(vq->dev, vq)) in vhost_net_tx_get_vq_desc()
472 r = vhost_get_vq_desc(vq, vq->iov, ARRAY_SIZE(vq->iov), in vhost_net_tx_get_vq_desc()
482 struct vhost_virtqueue *vq = &nvq->vq; in vhost_exceeds_maxpend() local
485 min_t(unsigned int, VHOST_MAX_PEND, vq->num >> 2); in vhost_exceeds_maxpend()
488 static size_t init_iov_iter(struct vhost_virtqueue *vq, struct iov_iter *iter, in init_iov_iter() argument
492 size_t len = iov_length(vq->iov, out); in init_iov_iter()
494 iov_iter_init(iter, WRITE, vq->iov, out, len); in init_iov_iter()
512 struct vhost_virtqueue *vq = &nvq->vq; in get_tx_bufs() local
517 if (ret < 0 || ret == vq->num) in get_tx_bufs()
521 vq_err(vq, "Unexpected descriptor format for TX: out %d, int %d\n", in get_tx_bufs()
527 *len = init_iov_iter(vq, &msg->msg_iter, nvq->vhost_hlen, *out); in get_tx_bufs()
529 vq_err(vq, "Unexpected header len for TX: %zd expected %zd\n", in get_tx_bufs()
537 static bool tx_can_batch(struct vhost_virtqueue *vq, size_t total_len) in tx_can_batch() argument
540 !vhost_vq_avail_empty(vq->dev, vq); in tx_can_batch()
546 struct vhost_virtqueue *vq = &nvq->vq; in handle_tx_copy() local
569 if (head == vq->num) { in handle_tx_copy()
571 vhost_poll_queue(&vq->poll); in handle_tx_copy()
573 vq))) { in handle_tx_copy()
574 vhost_disable_notify(&net->dev, vq); in handle_tx_copy()
580 vq->heads[nvq->done_idx].id = cpu_to_vhost32(vq, head); in handle_tx_copy()
581 vq->heads[nvq->done_idx].len = 0; in handle_tx_copy()
584 if (tx_can_batch(vq, total_len)) in handle_tx_copy()
592 vhost_discard_vq_desc(vq, 1); in handle_tx_copy()
593 vhost_net_enable_vq(net, vq); in handle_tx_copy()
602 vhost_poll_queue(&vq->poll); in handle_tx_copy()
613 struct vhost_virtqueue *vq = &nvq->vq; in handle_tx_zerocopy() local
633 vhost_zerocopy_signal_used(net, vq); in handle_tx_zerocopy()
642 if (head == vq->num) { in handle_tx_zerocopy()
644 vhost_poll_queue(&vq->poll); in handle_tx_zerocopy()
645 } else if (unlikely(vhost_enable_notify(&net->dev, vq))) { in handle_tx_zerocopy()
646 vhost_disable_notify(&net->dev, vq); in handle_tx_zerocopy()
661 vq->heads[nvq->upend_idx].id = cpu_to_vhost32(vq, head); in handle_tx_zerocopy()
662 vq->heads[nvq->upend_idx].len = VHOST_DMA_IN_PROGRESS; in handle_tx_zerocopy()
677 if (tx_can_batch(vq, total_len) && in handle_tx_zerocopy()
692 vhost_discard_vq_desc(vq, 1); in handle_tx_zerocopy()
693 vhost_net_enable_vq(net, vq); in handle_tx_zerocopy()
700 vhost_add_used_and_signal(&net->dev, vq, head, 0); in handle_tx_zerocopy()
702 vhost_zerocopy_signal_used(net, vq); in handle_tx_zerocopy()
705 vhost_poll_queue(&vq->poll); in handle_tx_zerocopy()
716 struct vhost_virtqueue *vq = &nvq->vq; in handle_tx() local
719 mutex_lock(&vq->mutex); in handle_tx()
720 sock = vq->private_data; in handle_tx()
724 if (!vq_iotlb_prefetch(vq)) in handle_tx()
727 vhost_disable_notify(&net->dev, vq); in handle_tx()
728 vhost_net_disable_vq(net, vq); in handle_tx()
736 mutex_unlock(&vq->mutex); in handle_tx()
775 struct vhost_virtqueue *rvq = &rnvq->vq; in vhost_net_rx_peek_head_len()
776 struct vhost_virtqueue *tvq = &tnvq->vq; in vhost_net_rx_peek_head_len()
829 static int get_rx_bufs(struct vhost_virtqueue *vq, in get_rx_bufs() argument
852 r = vhost_get_vq_desc(vq, vq->iov + seg, in get_rx_bufs()
853 ARRAY_SIZE(vq->iov) - seg, &out, in get_rx_bufs()
859 if (d == vq->num) { in get_rx_bufs()
864 vq_err(vq, "unexpected descriptor format for RX: " in get_rx_bufs()
873 heads[headcount].id = cpu_to_vhost32(vq, d); in get_rx_bufs()
874 len = iov_length(vq->iov + seg, in); in get_rx_bufs()
875 heads[headcount].len = cpu_to_vhost32(vq, len); in get_rx_bufs()
880 heads[headcount - 1].len = cpu_to_vhost32(vq, len + datalen); in get_rx_bufs()
892 vhost_discard_vq_desc(vq, headcount); in get_rx_bufs()
901 struct vhost_virtqueue *vq = &nvq->vq; in handle_rx() local
926 mutex_lock_nested(&vq->mutex, 0); in handle_rx()
927 sock = vq->private_data; in handle_rx()
931 if (!vq_iotlb_prefetch(vq)) in handle_rx()
934 vhost_disable_notify(&net->dev, vq); in handle_rx()
935 vhost_net_disable_vq(net, vq); in handle_rx()
940 vq_log = unlikely(vhost_has_feature(vq, VHOST_F_LOG_ALL)) ? in handle_rx()
941 vq->log : NULL; in handle_rx()
942 mergeable = vhost_has_feature(vq, VIRTIO_NET_F_MRG_RXBUF); in handle_rx()
948 headcount = get_rx_bufs(vq, vq->heads + nvq->done_idx, in handle_rx()
957 vhost_poll_queue(&vq->poll); in handle_rx()
958 } else if (unlikely(vhost_enable_notify(&net->dev, vq))) { in handle_rx()
961 vhost_disable_notify(&net->dev, vq); in handle_rx()
973 iov_iter_init(&msg.msg_iter, READ, vq->iov, 1, 1); in handle_rx()
980 iov_iter_init(&msg.msg_iter, READ, vq->iov, in, vhost_len); in handle_rx()
996 vhost_discard_vq_desc(vq, headcount); in handle_rx()
1003 vq_err(vq, "Unable to write vnet_hdr " in handle_rx()
1004 "at addr %p\n", vq->iov->iov_base); in handle_rx()
1015 num_buffers = cpu_to_vhost16(vq, headcount); in handle_rx()
1019 vq_err(vq, "Failed num_buffers write"); in handle_rx()
1020 vhost_discard_vq_desc(vq, headcount); in handle_rx()
1027 vhost_log_write(vq, vq_log, log, vhost_len); in handle_rx()
1030 vhost_poll_queue(&vq->poll); in handle_rx()
1035 vhost_poll_queue(&vq->poll); in handle_rx()
1037 vhost_net_enable_vq(net, vq); in handle_rx()
1040 mutex_unlock(&vq->mutex); in handle_rx()
1045 struct vhost_virtqueue *vq = container_of(work, struct vhost_virtqueue, in handle_tx_kick() local
1047 struct vhost_net *net = container_of(vq->dev, struct vhost_net, dev); in handle_tx_kick()
1054 struct vhost_virtqueue *vq = container_of(work, struct vhost_virtqueue, in handle_rx_kick() local
1056 struct vhost_net *net = container_of(vq->dev, struct vhost_net, dev); in handle_rx_kick()
1102 vqs[VHOST_NET_VQ_TX] = &n->vqs[VHOST_NET_VQ_TX].vq; in vhost_net_open()
1103 vqs[VHOST_NET_VQ_RX] = &n->vqs[VHOST_NET_VQ_RX].vq; in vhost_net_open()
1104 n->vqs[VHOST_NET_VQ_TX].vq.handle_kick = handle_tx_kick; in vhost_net_open()
1105 n->vqs[VHOST_NET_VQ_RX].vq.handle_kick = handle_rx_kick; in vhost_net_open()
1127 struct vhost_virtqueue *vq) in vhost_net_stop_vq() argument
1131 container_of(vq, struct vhost_net_virtqueue, vq); in vhost_net_stop_vq()
1133 mutex_lock(&vq->mutex); in vhost_net_stop_vq()
1134 sock = vq->private_data; in vhost_net_stop_vq()
1135 vhost_net_disable_vq(n, vq); in vhost_net_stop_vq()
1136 vq->private_data = NULL; in vhost_net_stop_vq()
1139 mutex_unlock(&vq->mutex); in vhost_net_stop_vq()
1146 *tx_sock = vhost_net_stop_vq(n, &n->vqs[VHOST_NET_VQ_TX].vq); in vhost_net_stop()
1147 *rx_sock = vhost_net_stop_vq(n, &n->vqs[VHOST_NET_VQ_RX].vq); in vhost_net_stop()
1153 vhost_poll_flush(&n->vqs[index].vq.poll); in vhost_net_flush_vq()
1161 mutex_lock(&n->vqs[VHOST_NET_VQ_TX].vq.mutex); in vhost_net_flush()
1163 mutex_unlock(&n->vqs[VHOST_NET_VQ_TX].vq.mutex); in vhost_net_flush()
1166 mutex_lock(&n->vqs[VHOST_NET_VQ_TX].vq.mutex); in vhost_net_flush()
1169 mutex_unlock(&n->vqs[VHOST_NET_VQ_TX].vq.mutex); in vhost_net_flush()
1285 struct vhost_virtqueue *vq; in vhost_net_set_backend() local
1299 vq = &n->vqs[index].vq; in vhost_net_set_backend()
1301 mutex_lock(&vq->mutex); in vhost_net_set_backend()
1304 if (!vhost_vq_access_ok(vq)) { in vhost_net_set_backend()
1315 oldsock = vq->private_data; in vhost_net_set_backend()
1317 ubufs = vhost_net_ubuf_alloc(vq, in vhost_net_set_backend()
1324 vhost_net_disable_vq(n, vq); in vhost_net_set_backend()
1325 vq->private_data = sock; in vhost_net_set_backend()
1327 r = vhost_vq_init_access(vq); in vhost_net_set_backend()
1330 r = vhost_net_enable_vq(n, vq); in vhost_net_set_backend()
1344 mutex_unlock(&vq->mutex); in vhost_net_set_backend()
1348 mutex_lock(&vq->mutex); in vhost_net_set_backend()
1349 vhost_zerocopy_signal_used(n, vq); in vhost_net_set_backend()
1350 mutex_unlock(&vq->mutex); in vhost_net_set_backend()
1362 vq->private_data = oldsock; in vhost_net_set_backend()
1363 vhost_net_enable_vq(n, vq); in vhost_net_set_backend()
1370 mutex_unlock(&vq->mutex); in vhost_net_set_backend()
1412 mutex_lock(&n->vqs[i].vq.mutex); in vhost_net_set_backend_features()
1413 n->vqs[i].vq.acked_backend_features = features; in vhost_net_set_backend_features()
1414 mutex_unlock(&n->vqs[i].vq.mutex); in vhost_net_set_backend_features()
1450 mutex_lock(&n->vqs[i].vq.mutex); in vhost_net_set_features()
1451 n->vqs[i].vq.acked_features = features; in vhost_net_set_features()
1454 mutex_unlock(&n->vqs[i].vq.mutex); in vhost_net_set_features()