Lines Matching refs:vq
49 #define vhost_used_event(vq) ((__virtio16 __user *)&vq->avail->ring[vq->num]) argument
50 #define vhost_avail_event(vq) ((__virtio16 __user *)&vq->used->ring[vq->num]) argument
53 static void vhost_disable_cross_endian(struct vhost_virtqueue *vq) in vhost_disable_cross_endian() argument
55 vq->user_be = !virtio_legacy_is_little_endian(); in vhost_disable_cross_endian()
58 static void vhost_enable_cross_endian_big(struct vhost_virtqueue *vq) in vhost_enable_cross_endian_big() argument
60 vq->user_be = true; in vhost_enable_cross_endian_big()
63 static void vhost_enable_cross_endian_little(struct vhost_virtqueue *vq) in vhost_enable_cross_endian_little() argument
65 vq->user_be = false; in vhost_enable_cross_endian_little()
68 static long vhost_set_vring_endian(struct vhost_virtqueue *vq, int __user *argp) in vhost_set_vring_endian() argument
72 if (vq->private_data) in vhost_set_vring_endian()
83 vhost_enable_cross_endian_big(vq); in vhost_set_vring_endian()
85 vhost_enable_cross_endian_little(vq); in vhost_set_vring_endian()
90 static long vhost_get_vring_endian(struct vhost_virtqueue *vq, u32 idx, in vhost_get_vring_endian() argument
95 .num = vq->user_be in vhost_get_vring_endian()
104 static void vhost_init_is_le(struct vhost_virtqueue *vq) in vhost_init_is_le() argument
111 vq->is_le = vhost_has_feature(vq, VIRTIO_F_VERSION_1) || !vq->user_be; in vhost_init_is_le()
114 static void vhost_disable_cross_endian(struct vhost_virtqueue *vq) in vhost_disable_cross_endian() argument
118 static long vhost_set_vring_endian(struct vhost_virtqueue *vq, int __user *argp) in vhost_set_vring_endian() argument
123 static long vhost_get_vring_endian(struct vhost_virtqueue *vq, u32 idx, in vhost_get_vring_endian() argument
129 static void vhost_init_is_le(struct vhost_virtqueue *vq) in vhost_init_is_le() argument
131 vq->is_le = vhost_has_feature(vq, VIRTIO_F_VERSION_1) in vhost_init_is_le()
136 static void vhost_reset_is_le(struct vhost_virtqueue *vq) in vhost_reset_is_le() argument
138 vhost_init_is_le(vq); in vhost_reset_is_le()
277 static void __vhost_vq_meta_reset(struct vhost_virtqueue *vq) in __vhost_vq_meta_reset() argument
282 vq->meta_iotlb[j] = NULL; in __vhost_vq_meta_reset()
299 bool vhost_vq_is_setup(struct vhost_virtqueue *vq) in vhost_vq_is_setup() argument
301 return vq->avail && vq->desc && vq->used && vhost_vq_access_ok(vq); in vhost_vq_is_setup()
306 struct vhost_virtqueue *vq) in vhost_vq_reset() argument
308 vq->num = 1; in vhost_vq_reset()
309 vq->desc = NULL; in vhost_vq_reset()
310 vq->avail = NULL; in vhost_vq_reset()
311 vq->used = NULL; in vhost_vq_reset()
312 vq->last_avail_idx = 0; in vhost_vq_reset()
313 vq->avail_idx = 0; in vhost_vq_reset()
314 vq->last_used_idx = 0; in vhost_vq_reset()
315 vq->signalled_used = 0; in vhost_vq_reset()
316 vq->signalled_used_valid = false; in vhost_vq_reset()
317 vq->used_flags = 0; in vhost_vq_reset()
318 vq->log_used = false; in vhost_vq_reset()
319 vq->log_addr = -1ull; in vhost_vq_reset()
320 vq->private_data = NULL; in vhost_vq_reset()
321 vq->acked_features = 0; in vhost_vq_reset()
322 vq->acked_backend_features = 0; in vhost_vq_reset()
323 vq->log_base = NULL; in vhost_vq_reset()
324 vq->error_ctx = NULL; in vhost_vq_reset()
325 vq->kick = NULL; in vhost_vq_reset()
326 vq->log_ctx = NULL; in vhost_vq_reset()
327 vhost_disable_cross_endian(vq); in vhost_vq_reset()
328 vhost_reset_is_le(vq); in vhost_vq_reset()
329 vq->busyloop_timeout = 0; in vhost_vq_reset()
330 vq->umem = NULL; in vhost_vq_reset()
331 vq->iotlb = NULL; in vhost_vq_reset()
332 vhost_vring_call_reset(&vq->call_ctx); in vhost_vq_reset()
333 __vhost_vq_meta_reset(vq); in vhost_vq_reset()
374 static void vhost_vq_free_iovecs(struct vhost_virtqueue *vq) in vhost_vq_free_iovecs() argument
376 kfree(vq->indirect); in vhost_vq_free_iovecs()
377 vq->indirect = NULL; in vhost_vq_free_iovecs()
378 kfree(vq->log); in vhost_vq_free_iovecs()
379 vq->log = NULL; in vhost_vq_free_iovecs()
380 kfree(vq->heads); in vhost_vq_free_iovecs()
381 vq->heads = NULL; in vhost_vq_free_iovecs()
387 struct vhost_virtqueue *vq; in vhost_dev_alloc_iovecs() local
391 vq = dev->vqs[i]; in vhost_dev_alloc_iovecs()
392 vq->indirect = kmalloc_array(UIO_MAXIOV, in vhost_dev_alloc_iovecs()
393 sizeof(*vq->indirect), in vhost_dev_alloc_iovecs()
395 vq->log = kmalloc_array(dev->iov_limit, sizeof(*vq->log), in vhost_dev_alloc_iovecs()
397 vq->heads = kmalloc_array(dev->iov_limit, sizeof(*vq->heads), in vhost_dev_alloc_iovecs()
399 if (!vq->indirect || !vq->log || !vq->heads) in vhost_dev_alloc_iovecs()
418 bool vhost_exceeds_weight(struct vhost_virtqueue *vq, in vhost_exceeds_weight() argument
421 struct vhost_dev *dev = vq->dev; in vhost_exceeds_weight()
425 vhost_poll_queue(&vq->poll); in vhost_exceeds_weight()
433 static size_t vhost_get_avail_size(struct vhost_virtqueue *vq, in vhost_get_avail_size() argument
437 vhost_has_feature(vq, VIRTIO_RING_F_EVENT_IDX) ? 2 : 0; in vhost_get_avail_size()
439 return sizeof(*vq->avail) + in vhost_get_avail_size()
440 sizeof(*vq->avail->ring) * num + event; in vhost_get_avail_size()
443 static size_t vhost_get_used_size(struct vhost_virtqueue *vq, in vhost_get_used_size() argument
447 vhost_has_feature(vq, VIRTIO_RING_F_EVENT_IDX) ? 2 : 0; in vhost_get_used_size()
449 return sizeof(*vq->used) + in vhost_get_used_size()
450 sizeof(*vq->used->ring) * num + event; in vhost_get_used_size()
453 static size_t vhost_get_desc_size(struct vhost_virtqueue *vq, in vhost_get_desc_size() argument
456 return sizeof(*vq->desc) * num; in vhost_get_desc_size()
466 struct vhost_virtqueue *vq; in vhost_dev_init() local
490 vq = dev->vqs[i]; in vhost_dev_init()
491 vq->log = NULL; in vhost_dev_init()
492 vq->indirect = NULL; in vhost_dev_init()
493 vq->heads = NULL; in vhost_dev_init()
494 vq->dev = dev; in vhost_dev_init()
495 mutex_init(&vq->mutex); in vhost_dev_init()
496 vhost_vq_reset(dev, vq); in vhost_dev_init()
497 if (vq->handle_kick) in vhost_dev_init()
498 vhost_poll_init(&vq->poll, vq->handle_kick, in vhost_dev_init()
768 static inline void __user *vhost_vq_meta_fetch(struct vhost_virtqueue *vq, in vhost_vq_meta_fetch() argument
772 const struct vhost_iotlb_map *map = vq->meta_iotlb[type]; in vhost_vq_meta_fetch()
806 static int translate_desc(struct vhost_virtqueue *vq, u64 addr, u32 len,
809 static int vhost_copy_to_user(struct vhost_virtqueue *vq, void __user *to, in vhost_copy_to_user() argument
814 if (!vq->iotlb) in vhost_copy_to_user()
823 void __user *uaddr = vhost_vq_meta_fetch(vq, in vhost_copy_to_user()
830 ret = translate_desc(vq, (u64)(uintptr_t)to, size, vq->iotlb_iov, in vhost_copy_to_user()
831 ARRAY_SIZE(vq->iotlb_iov), in vhost_copy_to_user()
835 iov_iter_init(&t, WRITE, vq->iotlb_iov, ret, size); in vhost_copy_to_user()
844 static int vhost_copy_from_user(struct vhost_virtqueue *vq, void *to, in vhost_copy_from_user() argument
849 if (!vq->iotlb) in vhost_copy_from_user()
857 void __user *uaddr = vhost_vq_meta_fetch(vq, in vhost_copy_from_user()
865 ret = translate_desc(vq, (u64)(uintptr_t)from, size, vq->iotlb_iov, in vhost_copy_from_user()
866 ARRAY_SIZE(vq->iotlb_iov), in vhost_copy_from_user()
869 vq_err(vq, "IOTLB translation failure: uaddr " in vhost_copy_from_user()
874 iov_iter_init(&f, READ, vq->iotlb_iov, ret, size); in vhost_copy_from_user()
884 static void __user *__vhost_get_user_slow(struct vhost_virtqueue *vq, in __vhost_get_user_slow() argument
890 ret = translate_desc(vq, (u64)(uintptr_t)addr, size, vq->iotlb_iov, in __vhost_get_user_slow()
891 ARRAY_SIZE(vq->iotlb_iov), in __vhost_get_user_slow()
894 vq_err(vq, "IOTLB translation failure: uaddr " in __vhost_get_user_slow()
900 if (ret != 1 || vq->iotlb_iov[0].iov_len != size) { in __vhost_get_user_slow()
901 vq_err(vq, "Non atomic userspace memory access: uaddr " in __vhost_get_user_slow()
907 return vq->iotlb_iov[0].iov_base; in __vhost_get_user_slow()
915 static inline void __user *__vhost_get_user(struct vhost_virtqueue *vq, in __vhost_get_user() argument
919 void __user *uaddr = vhost_vq_meta_fetch(vq, in __vhost_get_user()
924 return __vhost_get_user_slow(vq, addr, size, type); in __vhost_get_user()
927 #define vhost_put_user(vq, x, ptr) \ argument
930 if (!vq->iotlb) { \
934 (__typeof__(ptr)) __vhost_get_user(vq, ptr, \
944 static inline int vhost_put_avail_event(struct vhost_virtqueue *vq) in vhost_put_avail_event() argument
946 return vhost_put_user(vq, cpu_to_vhost16(vq, vq->avail_idx), in vhost_put_avail_event()
947 vhost_avail_event(vq)); in vhost_put_avail_event()
950 static inline int vhost_put_used(struct vhost_virtqueue *vq, in vhost_put_used() argument
954 return vhost_copy_to_user(vq, vq->used->ring + idx, head, in vhost_put_used()
958 static inline int vhost_put_used_flags(struct vhost_virtqueue *vq) in vhost_put_used_flags() argument
961 return vhost_put_user(vq, cpu_to_vhost16(vq, vq->used_flags), in vhost_put_used_flags()
962 &vq->used->flags); in vhost_put_used_flags()
965 static inline int vhost_put_used_idx(struct vhost_virtqueue *vq) in vhost_put_used_idx() argument
968 return vhost_put_user(vq, cpu_to_vhost16(vq, vq->last_used_idx), in vhost_put_used_idx()
969 &vq->used->idx); in vhost_put_used_idx()
972 #define vhost_get_user(vq, x, ptr, type) \ argument
975 if (!vq->iotlb) { \
979 (__typeof__(ptr)) __vhost_get_user(vq, ptr, \
990 #define vhost_get_avail(vq, x, ptr) \ argument
991 vhost_get_user(vq, x, ptr, VHOST_ADDR_AVAIL)
993 #define vhost_get_used(vq, x, ptr) \ argument
994 vhost_get_user(vq, x, ptr, VHOST_ADDR_USED)
1010 static inline int vhost_get_avail_idx(struct vhost_virtqueue *vq, in vhost_get_avail_idx() argument
1013 return vhost_get_avail(vq, *idx, &vq->avail->idx); in vhost_get_avail_idx()
1016 static inline int vhost_get_avail_head(struct vhost_virtqueue *vq, in vhost_get_avail_head() argument
1019 return vhost_get_avail(vq, *head, in vhost_get_avail_head()
1020 &vq->avail->ring[idx & (vq->num - 1)]); in vhost_get_avail_head()
1023 static inline int vhost_get_avail_flags(struct vhost_virtqueue *vq, in vhost_get_avail_flags() argument
1026 return vhost_get_avail(vq, *flags, &vq->avail->flags); in vhost_get_avail_flags()
1029 static inline int vhost_get_used_event(struct vhost_virtqueue *vq, in vhost_get_used_event() argument
1032 return vhost_get_avail(vq, *event, vhost_used_event(vq)); in vhost_get_used_event()
1035 static inline int vhost_get_used_idx(struct vhost_virtqueue *vq, in vhost_get_used_idx() argument
1038 return vhost_get_used(vq, *idx, &vq->used->idx); in vhost_get_used_idx()
1041 static inline int vhost_get_desc(struct vhost_virtqueue *vq, in vhost_get_desc() argument
1044 return vhost_copy_from_user(vq, desc, vq->desc + idx, sizeof(*desc)); in vhost_get_desc()
1059 vhost_poll_queue(&node->vq->poll); in vhost_iotlb_notify_vq()
1283 static int vhost_iotlb_miss(struct vhost_virtqueue *vq, u64 iova, int access) in vhost_iotlb_miss() argument
1285 struct vhost_dev *dev = vq->dev; in vhost_iotlb_miss()
1288 bool v2 = vhost_backend_has_feature(vq, VHOST_BACKEND_F_IOTLB_MSG_V2); in vhost_iotlb_miss()
1290 node = vhost_new_msg(vq, v2 ? VHOST_IOTLB_MSG_V2 : VHOST_IOTLB_MSG); in vhost_iotlb_miss()
1310 static bool vq_access_ok(struct vhost_virtqueue *vq, unsigned int num, in vq_access_ok() argument
1318 if (vq->iotlb) in vq_access_ok()
1321 return access_ok(desc, vhost_get_desc_size(vq, num)) && in vq_access_ok()
1322 access_ok(avail, vhost_get_avail_size(vq, num)) && in vq_access_ok()
1323 access_ok(used, vhost_get_used_size(vq, num)); in vq_access_ok()
1326 static void vhost_vq_meta_update(struct vhost_virtqueue *vq, in vhost_vq_meta_update() argument
1334 vq->meta_iotlb[type] = map; in vhost_vq_meta_update()
1337 static bool iotlb_access_ok(struct vhost_virtqueue *vq, in iotlb_access_ok() argument
1341 struct vhost_iotlb *umem = vq->iotlb; in iotlb_access_ok()
1344 if (vhost_vq_meta_fetch(vq, addr, len, type)) in iotlb_access_ok()
1350 vhost_iotlb_miss(vq, addr, access); in iotlb_access_ok()
1362 vhost_vq_meta_update(vq, map, type); in iotlb_access_ok()
1371 int vq_meta_prefetch(struct vhost_virtqueue *vq) in vq_meta_prefetch() argument
1373 unsigned int num = vq->num; in vq_meta_prefetch()
1375 if (!vq->iotlb) in vq_meta_prefetch()
1378 return iotlb_access_ok(vq, VHOST_MAP_RO, (u64)(uintptr_t)vq->desc, in vq_meta_prefetch()
1379 vhost_get_desc_size(vq, num), VHOST_ADDR_DESC) && in vq_meta_prefetch()
1380 iotlb_access_ok(vq, VHOST_MAP_RO, (u64)(uintptr_t)vq->avail, in vq_meta_prefetch()
1381 vhost_get_avail_size(vq, num), in vq_meta_prefetch()
1383 iotlb_access_ok(vq, VHOST_MAP_WO, (u64)(uintptr_t)vq->used, in vq_meta_prefetch()
1384 vhost_get_used_size(vq, num), VHOST_ADDR_USED); in vq_meta_prefetch()
1396 static bool vq_log_used_access_ok(struct vhost_virtqueue *vq, in vq_log_used_access_ok() argument
1403 if (vq->iotlb) in vq_log_used_access_ok()
1407 vhost_get_used_size(vq, vq->num)); in vq_log_used_access_ok()
1412 static bool vq_log_access_ok(struct vhost_virtqueue *vq, in vq_log_access_ok() argument
1415 return vq_memory_access_ok(log_base, vq->umem, in vq_log_access_ok()
1416 vhost_has_feature(vq, VHOST_F_LOG_ALL)) && in vq_log_access_ok()
1417 vq_log_used_access_ok(vq, log_base, vq->log_used, vq->log_addr); in vq_log_access_ok()
1422 bool vhost_vq_access_ok(struct vhost_virtqueue *vq) in vhost_vq_access_ok() argument
1424 if (!vq_log_access_ok(vq, vq->log_base)) in vhost_vq_access_ok()
1427 return vq_access_ok(vq, vq->num, vq->desc, vq->avail, vq->used); in vhost_vq_access_ok()
1499 struct vhost_virtqueue *vq, in vhost_vring_set_num() argument
1506 if (vq->private_data) in vhost_vring_set_num()
1514 vq->num = s.num; in vhost_vring_set_num()
1520 struct vhost_virtqueue *vq, in vhost_vring_set_addr() argument
1538 BUILD_BUG_ON(__alignof__ *vq->avail > VRING_AVAIL_ALIGN_SIZE); in vhost_vring_set_addr()
1539 BUILD_BUG_ON(__alignof__ *vq->used > VRING_USED_ALIGN_SIZE); in vhost_vring_set_addr()
1548 if (vq->private_data) { in vhost_vring_set_addr()
1549 if (!vq_access_ok(vq, vq->num, in vhost_vring_set_addr()
1556 if (!vq_log_used_access_ok(vq, vq->log_base, in vhost_vring_set_addr()
1562 vq->log_used = !!(a.flags & (0x1 << VHOST_VRING_F_LOG)); in vhost_vring_set_addr()
1563 vq->desc = (void __user *)(unsigned long)a.desc_user_addr; in vhost_vring_set_addr()
1564 vq->avail = (void __user *)(unsigned long)a.avail_user_addr; in vhost_vring_set_addr()
1565 vq->log_addr = a.log_guest_addr; in vhost_vring_set_addr()
1566 vq->used = (void __user *)(unsigned long)a.used_user_addr; in vhost_vring_set_addr()
1572 struct vhost_virtqueue *vq, in vhost_vring_set_num_addr() argument
1578 mutex_lock(&vq->mutex); in vhost_vring_set_num_addr()
1582 r = vhost_vring_set_num(d, vq, argp); in vhost_vring_set_num_addr()
1585 r = vhost_vring_set_addr(d, vq, argp); in vhost_vring_set_num_addr()
1591 mutex_unlock(&vq->mutex); in vhost_vring_set_num_addr()
1601 struct vhost_virtqueue *vq; in vhost_vring_ioctl() local
1614 vq = d->vqs[idx]; in vhost_vring_ioctl()
1618 return vhost_vring_set_num_addr(d, vq, ioctl, argp); in vhost_vring_ioctl()
1621 mutex_lock(&vq->mutex); in vhost_vring_ioctl()
1627 if (vq->private_data) { in vhost_vring_ioctl()
1639 vq->last_avail_idx = s.num; in vhost_vring_ioctl()
1641 vq->avail_idx = vq->last_avail_idx; in vhost_vring_ioctl()
1645 s.num = vq->last_avail_idx; in vhost_vring_ioctl()
1659 if (eventfp != vq->kick) { in vhost_vring_ioctl()
1660 pollstop = (filep = vq->kick) != NULL; in vhost_vring_ioctl()
1661 pollstart = (vq->kick = eventfp) != NULL; in vhost_vring_ioctl()
1676 swap(ctx, vq->call_ctx.ctx); in vhost_vring_ioctl()
1688 swap(ctx, vq->error_ctx); in vhost_vring_ioctl()
1691 r = vhost_set_vring_endian(vq, argp); in vhost_vring_ioctl()
1694 r = vhost_get_vring_endian(vq, idx, argp); in vhost_vring_ioctl()
1701 vq->busyloop_timeout = s.num; in vhost_vring_ioctl()
1705 s.num = vq->busyloop_timeout; in vhost_vring_ioctl()
1713 if (pollstop && vq->handle_kick) in vhost_vring_ioctl()
1714 vhost_poll_stop(&vq->poll); in vhost_vring_ioctl()
1721 if (pollstart && vq->handle_kick) in vhost_vring_ioctl()
1722 r = vhost_poll_start(&vq->poll, vq->kick); in vhost_vring_ioctl()
1724 mutex_unlock(&vq->mutex); in vhost_vring_ioctl()
1726 if (pollstop && vq->handle_kick) in vhost_vring_ioctl()
1727 vhost_dev_flush(vq->poll.dev); in vhost_vring_ioctl()
1745 struct vhost_virtqueue *vq = d->vqs[i]; in vhost_init_device_iotlb() local
1747 mutex_lock(&vq->mutex); in vhost_init_device_iotlb()
1748 vq->iotlb = niotlb; in vhost_init_device_iotlb()
1749 __vhost_vq_meta_reset(vq); in vhost_init_device_iotlb()
1750 mutex_unlock(&vq->mutex); in vhost_init_device_iotlb()
1792 struct vhost_virtqueue *vq; in vhost_dev_ioctl() local
1794 vq = d->vqs[i]; in vhost_dev_ioctl()
1795 mutex_lock(&vq->mutex); in vhost_dev_ioctl()
1797 if (vq->private_data && !vq_log_access_ok(vq, base)) in vhost_dev_ioctl()
1800 vq->log_base = base; in vhost_dev_ioctl()
1801 mutex_unlock(&vq->mutex); in vhost_dev_ioctl()
1880 static int log_write_hva(struct vhost_virtqueue *vq, u64 hva, u64 len) in log_write_hva() argument
1882 struct vhost_iotlb *umem = vq->umem; in log_write_hva()
1900 r = log_write(vq->log_base, in log_write_hva()
1919 static int log_used(struct vhost_virtqueue *vq, u64 used_offset, u64 len) in log_used() argument
1921 struct iovec *iov = vq->log_iov; in log_used()
1924 if (!vq->iotlb) in log_used()
1925 return log_write(vq->log_base, vq->log_addr + used_offset, len); in log_used()
1927 ret = translate_desc(vq, (uintptr_t)vq->used + used_offset, in log_used()
1933 ret = log_write_hva(vq, (uintptr_t)iov[i].iov_base, in log_used()
1942 int vhost_log_write(struct vhost_virtqueue *vq, struct vhost_log *log, in vhost_log_write() argument
1950 if (vq->iotlb) { in vhost_log_write()
1952 r = log_write_hva(vq, (uintptr_t)iov[i].iov_base, in vhost_log_write()
1962 r = log_write(vq->log_base, log[i].addr, l); in vhost_log_write()
1967 if (vq->log_ctx) in vhost_log_write()
1968 eventfd_signal(vq->log_ctx, 1); in vhost_log_write()
1978 static int vhost_update_used_flags(struct vhost_virtqueue *vq) in vhost_update_used_flags() argument
1981 if (vhost_put_used_flags(vq)) in vhost_update_used_flags()
1983 if (unlikely(vq->log_used)) { in vhost_update_used_flags()
1987 used = &vq->used->flags; in vhost_update_used_flags()
1988 log_used(vq, (used - (void __user *)vq->used), in vhost_update_used_flags()
1989 sizeof vq->used->flags); in vhost_update_used_flags()
1990 if (vq->log_ctx) in vhost_update_used_flags()
1991 eventfd_signal(vq->log_ctx, 1); in vhost_update_used_flags()
1996 static int vhost_update_avail_event(struct vhost_virtqueue *vq) in vhost_update_avail_event() argument
1998 if (vhost_put_avail_event(vq)) in vhost_update_avail_event()
2000 if (unlikely(vq->log_used)) { in vhost_update_avail_event()
2005 used = vhost_avail_event(vq); in vhost_update_avail_event()
2006 log_used(vq, (used - (void __user *)vq->used), in vhost_update_avail_event()
2007 sizeof *vhost_avail_event(vq)); in vhost_update_avail_event()
2008 if (vq->log_ctx) in vhost_update_avail_event()
2009 eventfd_signal(vq->log_ctx, 1); in vhost_update_avail_event()
2014 int vhost_vq_init_access(struct vhost_virtqueue *vq) in vhost_vq_init_access() argument
2018 bool is_le = vq->is_le; in vhost_vq_init_access()
2020 if (!vq->private_data) in vhost_vq_init_access()
2023 vhost_init_is_le(vq); in vhost_vq_init_access()
2025 r = vhost_update_used_flags(vq); in vhost_vq_init_access()
2028 vq->signalled_used_valid = false; in vhost_vq_init_access()
2029 if (!vq->iotlb && in vhost_vq_init_access()
2030 !access_ok(&vq->used->idx, sizeof vq->used->idx)) { in vhost_vq_init_access()
2034 r = vhost_get_used_idx(vq, &last_used_idx); in vhost_vq_init_access()
2036 vq_err(vq, "Can't access used idx at %p\n", in vhost_vq_init_access()
2037 &vq->used->idx); in vhost_vq_init_access()
2040 vq->last_used_idx = vhost16_to_cpu(vq, last_used_idx); in vhost_vq_init_access()
2044 vq->is_le = is_le; in vhost_vq_init_access()
2049 static int translate_desc(struct vhost_virtqueue *vq, u64 addr, u32 len, in translate_desc() argument
2053 struct vhost_dev *dev = vq->dev; in translate_desc()
2090 vhost_iotlb_miss(vq, addr, access); in translate_desc()
2097 static unsigned next_desc(struct vhost_virtqueue *vq, struct vring_desc *desc) in next_desc() argument
2102 if (!(desc->flags & cpu_to_vhost16(vq, VRING_DESC_F_NEXT))) in next_desc()
2106 next = vhost16_to_cpu(vq, READ_ONCE(desc->next)); in next_desc()
2110 static int get_indirect(struct vhost_virtqueue *vq, in get_indirect() argument
2118 u32 len = vhost32_to_cpu(vq, indirect->len); in get_indirect()
2124 vq_err(vq, "Invalid length in indirect descriptor: " in get_indirect()
2131 ret = translate_desc(vq, vhost64_to_cpu(vq, indirect->addr), len, vq->indirect, in get_indirect()
2135 vq_err(vq, "Translation failure %d in indirect.\n", ret); in get_indirect()
2138 iov_iter_init(&from, READ, vq->indirect, ret, len); in get_indirect()
2143 vq_err(vq, "Indirect buffer length too big: %d\n", in get_indirect()
2151 vq_err(vq, "Loop detected: last one at %u " in get_indirect()
2157 vq_err(vq, "Failed indirect descriptor: idx %d, %zx\n", in get_indirect()
2158 i, (size_t)vhost64_to_cpu(vq, indirect->addr) + i * sizeof desc); in get_indirect()
2161 if (unlikely(desc.flags & cpu_to_vhost16(vq, VRING_DESC_F_INDIRECT))) { in get_indirect()
2162 vq_err(vq, "Nested indirect descriptor: idx %d, %zx\n", in get_indirect()
2163 i, (size_t)vhost64_to_cpu(vq, indirect->addr) + i * sizeof desc); in get_indirect()
2167 if (desc.flags & cpu_to_vhost16(vq, VRING_DESC_F_WRITE)) in get_indirect()
2172 ret = translate_desc(vq, vhost64_to_cpu(vq, desc.addr), in get_indirect()
2173 vhost32_to_cpu(vq, desc.len), iov + iov_count, in get_indirect()
2177 vq_err(vq, "Translation failure %d indirect idx %d\n", in get_indirect()
2185 log[*log_num].addr = vhost64_to_cpu(vq, desc.addr); in get_indirect()
2186 log[*log_num].len = vhost32_to_cpu(vq, desc.len); in get_indirect()
2193 vq_err(vq, "Indirect descriptor " in get_indirect()
2199 } while ((i = next_desc(vq, &desc)) != -1); in get_indirect()
2211 int vhost_get_vq_desc(struct vhost_virtqueue *vq, in vhost_get_vq_desc() argument
2224 last_avail_idx = vq->last_avail_idx; in vhost_get_vq_desc()
2226 if (vq->avail_idx == vq->last_avail_idx) { in vhost_get_vq_desc()
2227 if (unlikely(vhost_get_avail_idx(vq, &avail_idx))) { in vhost_get_vq_desc()
2228 vq_err(vq, "Failed to access avail idx at %p\n", in vhost_get_vq_desc()
2229 &vq->avail->idx); in vhost_get_vq_desc()
2232 vq->avail_idx = vhost16_to_cpu(vq, avail_idx); in vhost_get_vq_desc()
2234 if (unlikely((u16)(vq->avail_idx - last_avail_idx) > vq->num)) { in vhost_get_vq_desc()
2235 vq_err(vq, "Guest moved used index from %u to %u", in vhost_get_vq_desc()
2236 last_avail_idx, vq->avail_idx); in vhost_get_vq_desc()
2243 if (vq->avail_idx == last_avail_idx) in vhost_get_vq_desc()
2244 return vq->num; in vhost_get_vq_desc()
2254 if (unlikely(vhost_get_avail_head(vq, &ring_head, last_avail_idx))) { in vhost_get_vq_desc()
2255 vq_err(vq, "Failed to read head: idx %d address %p\n", in vhost_get_vq_desc()
2257 &vq->avail->ring[last_avail_idx % vq->num]); in vhost_get_vq_desc()
2261 head = vhost16_to_cpu(vq, ring_head); in vhost_get_vq_desc()
2264 if (unlikely(head >= vq->num)) { in vhost_get_vq_desc()
2265 vq_err(vq, "Guest says index %u > %u is available", in vhost_get_vq_desc()
2266 head, vq->num); in vhost_get_vq_desc()
2278 if (unlikely(i >= vq->num)) { in vhost_get_vq_desc()
2279 vq_err(vq, "Desc index is %u > %u, head = %u", in vhost_get_vq_desc()
2280 i, vq->num, head); in vhost_get_vq_desc()
2283 if (unlikely(++found > vq->num)) { in vhost_get_vq_desc()
2284 vq_err(vq, "Loop detected: last one at %u " in vhost_get_vq_desc()
2286 i, vq->num, head); in vhost_get_vq_desc()
2289 ret = vhost_get_desc(vq, &desc, i); in vhost_get_vq_desc()
2291 vq_err(vq, "Failed to get descriptor: idx %d addr %p\n", in vhost_get_vq_desc()
2292 i, vq->desc + i); in vhost_get_vq_desc()
2295 if (desc.flags & cpu_to_vhost16(vq, VRING_DESC_F_INDIRECT)) { in vhost_get_vq_desc()
2296 ret = get_indirect(vq, iov, iov_size, in vhost_get_vq_desc()
2301 vq_err(vq, "Failure detected " in vhost_get_vq_desc()
2308 if (desc.flags & cpu_to_vhost16(vq, VRING_DESC_F_WRITE)) in vhost_get_vq_desc()
2312 ret = translate_desc(vq, vhost64_to_cpu(vq, desc.addr), in vhost_get_vq_desc()
2313 vhost32_to_cpu(vq, desc.len), iov + iov_count, in vhost_get_vq_desc()
2317 vq_err(vq, "Translation failure %d descriptor idx %d\n", in vhost_get_vq_desc()
2326 log[*log_num].addr = vhost64_to_cpu(vq, desc.addr); in vhost_get_vq_desc()
2327 log[*log_num].len = vhost32_to_cpu(vq, desc.len); in vhost_get_vq_desc()
2334 vq_err(vq, "Descriptor has out after in: " in vhost_get_vq_desc()
2340 } while ((i = next_desc(vq, &desc)) != -1); in vhost_get_vq_desc()
2343 vq->last_avail_idx++; in vhost_get_vq_desc()
2347 BUG_ON(!(vq->used_flags & VRING_USED_F_NO_NOTIFY)); in vhost_get_vq_desc()
2353 void vhost_discard_vq_desc(struct vhost_virtqueue *vq, int n) in vhost_discard_vq_desc() argument
2355 vq->last_avail_idx -= n; in vhost_discard_vq_desc()
2361 int vhost_add_used(struct vhost_virtqueue *vq, unsigned int head, int len) in vhost_add_used() argument
2364 cpu_to_vhost32(vq, head), in vhost_add_used()
2365 cpu_to_vhost32(vq, len) in vhost_add_used()
2368 return vhost_add_used_n(vq, &heads, 1); in vhost_add_used()
2372 static int __vhost_add_used_n(struct vhost_virtqueue *vq, in __vhost_add_used_n() argument
2380 start = vq->last_used_idx & (vq->num - 1); in __vhost_add_used_n()
2381 used = vq->used->ring + start; in __vhost_add_used_n()
2382 if (vhost_put_used(vq, heads, start, count)) { in __vhost_add_used_n()
2383 vq_err(vq, "Failed to write used"); in __vhost_add_used_n()
2386 if (unlikely(vq->log_used)) { in __vhost_add_used_n()
2390 log_used(vq, ((void __user *)used - (void __user *)vq->used), in __vhost_add_used_n()
2393 old = vq->last_used_idx; in __vhost_add_used_n()
2394 new = (vq->last_used_idx += count); in __vhost_add_used_n()
2399 if (unlikely((u16)(new - vq->signalled_used) < (u16)(new - old))) in __vhost_add_used_n()
2400 vq->signalled_used_valid = false; in __vhost_add_used_n()
2406 int vhost_add_used_n(struct vhost_virtqueue *vq, struct vring_used_elem *heads, in vhost_add_used_n() argument
2411 start = vq->last_used_idx & (vq->num - 1); in vhost_add_used_n()
2412 n = vq->num - start; in vhost_add_used_n()
2414 r = __vhost_add_used_n(vq, heads, n); in vhost_add_used_n()
2420 r = __vhost_add_used_n(vq, heads, count); in vhost_add_used_n()
2424 if (vhost_put_used_idx(vq)) { in vhost_add_used_n()
2425 vq_err(vq, "Failed to increment used idx"); in vhost_add_used_n()
2428 if (unlikely(vq->log_used)) { in vhost_add_used_n()
2432 log_used(vq, offsetof(struct vring_used, idx), in vhost_add_used_n()
2433 sizeof vq->used->idx); in vhost_add_used_n()
2434 if (vq->log_ctx) in vhost_add_used_n()
2435 eventfd_signal(vq->log_ctx, 1); in vhost_add_used_n()
2441 static bool vhost_notify(struct vhost_dev *dev, struct vhost_virtqueue *vq) in vhost_notify() argument
2451 if (vhost_has_feature(vq, VIRTIO_F_NOTIFY_ON_EMPTY) && in vhost_notify()
2452 unlikely(vq->avail_idx == vq->last_avail_idx)) in vhost_notify()
2455 if (!vhost_has_feature(vq, VIRTIO_RING_F_EVENT_IDX)) { in vhost_notify()
2457 if (vhost_get_avail_flags(vq, &flags)) { in vhost_notify()
2458 vq_err(vq, "Failed to get flags"); in vhost_notify()
2461 return !(flags & cpu_to_vhost16(vq, VRING_AVAIL_F_NO_INTERRUPT)); in vhost_notify()
2463 old = vq->signalled_used; in vhost_notify()
2464 v = vq->signalled_used_valid; in vhost_notify()
2465 new = vq->signalled_used = vq->last_used_idx; in vhost_notify()
2466 vq->signalled_used_valid = true; in vhost_notify()
2471 if (vhost_get_used_event(vq, &event)) { in vhost_notify()
2472 vq_err(vq, "Failed to get used event idx"); in vhost_notify()
2475 return vring_need_event(vhost16_to_cpu(vq, event), new, old); in vhost_notify()
2479 void vhost_signal(struct vhost_dev *dev, struct vhost_virtqueue *vq) in vhost_signal() argument
2482 if (vq->call_ctx.ctx && vhost_notify(dev, vq)) in vhost_signal()
2483 eventfd_signal(vq->call_ctx.ctx, 1); in vhost_signal()
2489 struct vhost_virtqueue *vq, in vhost_add_used_and_signal() argument
2492 vhost_add_used(vq, head, len); in vhost_add_used_and_signal()
2493 vhost_signal(dev, vq); in vhost_add_used_and_signal()
2499 struct vhost_virtqueue *vq, in vhost_add_used_and_signal_n() argument
2502 vhost_add_used_n(vq, heads, count); in vhost_add_used_and_signal_n()
2503 vhost_signal(dev, vq); in vhost_add_used_and_signal_n()
2508 bool vhost_vq_avail_empty(struct vhost_dev *dev, struct vhost_virtqueue *vq) in vhost_vq_avail_empty() argument
2513 if (vq->avail_idx != vq->last_avail_idx) in vhost_vq_avail_empty()
2516 r = vhost_get_avail_idx(vq, &avail_idx); in vhost_vq_avail_empty()
2519 vq->avail_idx = vhost16_to_cpu(vq, avail_idx); in vhost_vq_avail_empty()
2521 return vq->avail_idx == vq->last_avail_idx; in vhost_vq_avail_empty()
2526 bool vhost_enable_notify(struct vhost_dev *dev, struct vhost_virtqueue *vq) in vhost_enable_notify() argument
2531 if (!(vq->used_flags & VRING_USED_F_NO_NOTIFY)) in vhost_enable_notify()
2533 vq->used_flags &= ~VRING_USED_F_NO_NOTIFY; in vhost_enable_notify()
2534 if (!vhost_has_feature(vq, VIRTIO_RING_F_EVENT_IDX)) { in vhost_enable_notify()
2535 r = vhost_update_used_flags(vq); in vhost_enable_notify()
2537 vq_err(vq, "Failed to enable notification at %p: %d\n", in vhost_enable_notify()
2538 &vq->used->flags, r); in vhost_enable_notify()
2542 r = vhost_update_avail_event(vq); in vhost_enable_notify()
2544 vq_err(vq, "Failed to update avail event index at %p: %d\n", in vhost_enable_notify()
2545 vhost_avail_event(vq), r); in vhost_enable_notify()
2552 r = vhost_get_avail_idx(vq, &avail_idx); in vhost_enable_notify()
2554 vq_err(vq, "Failed to check avail idx at %p: %d\n", in vhost_enable_notify()
2555 &vq->avail->idx, r); in vhost_enable_notify()
2558 vq->avail_idx = vhost16_to_cpu(vq, avail_idx); in vhost_enable_notify()
2560 return vq->avail_idx != vq->last_avail_idx; in vhost_enable_notify()
2565 void vhost_disable_notify(struct vhost_dev *dev, struct vhost_virtqueue *vq) in vhost_disable_notify() argument
2569 if (vq->used_flags & VRING_USED_F_NO_NOTIFY) in vhost_disable_notify()
2571 vq->used_flags |= VRING_USED_F_NO_NOTIFY; in vhost_disable_notify()
2572 if (!vhost_has_feature(vq, VIRTIO_RING_F_EVENT_IDX)) { in vhost_disable_notify()
2573 r = vhost_update_used_flags(vq); in vhost_disable_notify()
2575 vq_err(vq, "Failed to disable notification at %p: %d\n", in vhost_disable_notify()
2576 &vq->used->flags, r); in vhost_disable_notify()
2582 struct vhost_msg_node *vhost_new_msg(struct vhost_virtqueue *vq, int type) in vhost_new_msg() argument
2590 node->vq = vq; in vhost_new_msg()
2626 struct vhost_virtqueue *vq; in vhost_set_backend_features() local
2631 vq = dev->vqs[i]; in vhost_set_backend_features()
2632 mutex_lock(&vq->mutex); in vhost_set_backend_features()
2633 vq->acked_backend_features = features; in vhost_set_backend_features()
2634 mutex_unlock(&vq->mutex); in vhost_set_backend_features()