Lines Matching refs:vrh
38 static inline int __vringh_get_head(const struct vringh *vrh, in __vringh_get_head() argument
39 int (*getu16)(const struct vringh *vrh, in __vringh_get_head() argument
46 err = getu16(vrh, &avail_idx, &vrh->vring.avail->idx); in __vringh_get_head()
49 &vrh->vring.avail->idx); in __vringh_get_head()
54 return vrh->vring.num; in __vringh_get_head()
57 virtio_rmb(vrh->weak_barriers); in __vringh_get_head()
59 i = *last_avail_idx & (vrh->vring.num - 1); in __vringh_get_head()
61 err = getu16(vrh, &head, &vrh->vring.avail->ring[i]); in __vringh_get_head()
64 *last_avail_idx, &vrh->vring.avail->ring[i]); in __vringh_get_head()
68 if (head >= vrh->vring.num) { in __vringh_get_head()
70 head, vrh->vring.num); in __vringh_get_head()
79 static inline ssize_t vringh_iov_xfer(struct vringh *vrh, in vringh_iov_xfer() argument
82 int (*xfer)(const struct vringh *vrh, in vringh_iov_xfer() argument
92 err = xfer(vrh, iov->iov[iov->i].iov_base, ptr, partlen); in vringh_iov_xfer()
116 static inline bool range_check(struct vringh *vrh, u64 addr, size_t *len, in range_check() argument
122 if (!getrange(vrh, addr, range)) in range_check()
150 static inline bool no_range_check(struct vringh *vrh, u64 addr, size_t *len, in no_range_check() argument
159 static int move_to_indirect(const struct vringh *vrh, in move_to_indirect() argument
172 len = vringh32_to_cpu(vrh, desc->len); in move_to_indirect()
179 if (desc->flags & cpu_to_vringh16(vrh, VRING_DESC_F_NEXT)) in move_to_indirect()
180 *up_next = vringh16_to_cpu(vrh, desc->next); in move_to_indirect()
217 static u16 __cold return_from_indirect(const struct vringh *vrh, int *up_next, in return_from_indirect() argument
223 *descs = vrh->vring.desc; in return_from_indirect()
224 *desc_max = vrh->vring.num; in return_from_indirect()
228 static int slow_copy(struct vringh *vrh, void *dst, const void *src, in slow_copy() argument
229 bool (*rcheck)(struct vringh *vrh, u64 addr, size_t *len, in slow_copy() argument
231 bool (*getrange)(struct vringh *vrh, in slow_copy() argument
234 bool (*getrange)(struct vringh *vrh, in slow_copy()
238 int (*copy)(const struct vringh *vrh, in slow_copy() argument
250 if (!rcheck(vrh, addr, &part, range, getrange)) in slow_copy()
253 err = copy(vrh, dst, src, part); in slow_copy()
265 __vringh_iov(struct vringh *vrh, u16 i, in __vringh_iov() argument
268 bool (*rcheck)(struct vringh *vrh, u64 addr, size_t *len, in __vringh_iov() argument
274 int (*copy)(const struct vringh *vrh, in __vringh_iov() argument
283 descs = vrh->vring.desc; in __vringh_iov()
284 desc_max = vrh->vring.num; in __vringh_iov()
302 err = slow_copy(vrh, &desc, &descs[i], rcheck, getrange, in __vringh_iov()
305 err = copy(vrh, &desc, &descs[i], sizeof(desc)); in __vringh_iov()
310 cpu_to_vringh16(vrh, VRING_DESC_F_INDIRECT))) { in __vringh_iov()
311 u64 a = vringh64_to_cpu(vrh, desc.addr); in __vringh_iov()
314 len = vringh32_to_cpu(vrh, desc.len); in __vringh_iov()
315 if (!rcheck(vrh, a, &len, &range, getrange)) { in __vringh_iov()
320 if (unlikely(len != vringh32_to_cpu(vrh, desc.len))) { in __vringh_iov()
327 err = move_to_indirect(vrh, &up_next, &i, addr, &desc, in __vringh_iov()
334 if (count++ == vrh->vring.num) { in __vringh_iov()
340 if (desc.flags & cpu_to_vringh16(vrh, VRING_DESC_F_WRITE)) in __vringh_iov()
361 len = vringh32_to_cpu(vrh, desc.len); in __vringh_iov()
362 if (!rcheck(vrh, vringh64_to_cpu(vrh, desc.addr), &len, &range, in __vringh_iov()
367 addr = (void *)(unsigned long)(vringh64_to_cpu(vrh, desc.addr) + in __vringh_iov()
380 if (unlikely(len != vringh32_to_cpu(vrh, desc.len))) { in __vringh_iov()
381 desc.len = cpu_to_vringh32(vrh, in __vringh_iov()
382 vringh32_to_cpu(vrh, desc.len) - len); in __vringh_iov()
383 desc.addr = cpu_to_vringh64(vrh, in __vringh_iov()
384 vringh64_to_cpu(vrh, desc.addr) + len); in __vringh_iov()
388 if (desc.flags & cpu_to_vringh16(vrh, VRING_DESC_F_NEXT)) { in __vringh_iov()
389 i = vringh16_to_cpu(vrh, desc.next); in __vringh_iov()
393 i = return_from_indirect(vrh, &up_next, in __vringh_iov()
413 static inline int __vringh_complete(struct vringh *vrh, in __vringh_complete() argument
416 int (*putu16)(const struct vringh *vrh, in __vringh_complete() argument
418 int (*putused)(const struct vringh *vrh, in __vringh_complete() argument
427 used_ring = vrh->vring.used; in __vringh_complete()
428 used_idx = vrh->last_used_idx + vrh->completed; in __vringh_complete()
430 off = used_idx % vrh->vring.num; in __vringh_complete()
433 if (num_used > 1 && unlikely(off + num_used >= vrh->vring.num)) { in __vringh_complete()
434 u16 part = vrh->vring.num - off; in __vringh_complete()
435 err = putused(vrh, &used_ring->ring[off], used, part); in __vringh_complete()
437 err = putused(vrh, &used_ring->ring[0], used + part, in __vringh_complete()
440 err = putused(vrh, &used_ring->ring[off], used, num_used); in __vringh_complete()
449 virtio_wmb(vrh->weak_barriers); in __vringh_complete()
451 err = putu16(vrh, &vrh->vring.used->idx, used_idx + num_used); in __vringh_complete()
454 &vrh->vring.used->idx); in __vringh_complete()
458 vrh->completed += num_used; in __vringh_complete()
463 static inline int __vringh_need_notify(struct vringh *vrh, in __vringh_need_notify() argument
464 int (*getu16)(const struct vringh *vrh, in __vringh_need_notify() argument
475 virtio_mb(vrh->weak_barriers); in __vringh_need_notify()
478 if (!vrh->event_indices) { in __vringh_need_notify()
480 err = getu16(vrh, &flags, &vrh->vring.avail->flags); in __vringh_need_notify()
483 &vrh->vring.avail->flags); in __vringh_need_notify()
490 err = getu16(vrh, &used_event, &vring_used_event(&vrh->vring)); in __vringh_need_notify()
493 &vring_used_event(&vrh->vring)); in __vringh_need_notify()
498 if (unlikely(vrh->completed > 0xffff)) in __vringh_need_notify()
502 vrh->last_used_idx + vrh->completed, in __vringh_need_notify()
503 vrh->last_used_idx); in __vringh_need_notify()
505 vrh->last_used_idx += vrh->completed; in __vringh_need_notify()
506 vrh->completed = 0; in __vringh_need_notify()
510 static inline bool __vringh_notify_enable(struct vringh *vrh, in __vringh_notify_enable() argument
511 int (*getu16)(const struct vringh *vrh, in __vringh_notify_enable() argument
513 int (*putu16)(const struct vringh *vrh, in __vringh_notify_enable() argument
518 if (!vrh->event_indices) { in __vringh_notify_enable()
520 if (putu16(vrh, &vrh->vring.used->flags, 0) != 0) { in __vringh_notify_enable()
522 &vrh->vring.used->flags); in __vringh_notify_enable()
526 if (putu16(vrh, &vring_avail_event(&vrh->vring), in __vringh_notify_enable()
527 vrh->last_avail_idx) != 0) { in __vringh_notify_enable()
529 &vring_avail_event(&vrh->vring)); in __vringh_notify_enable()
536 virtio_mb(vrh->weak_barriers); in __vringh_notify_enable()
538 if (getu16(vrh, &avail, &vrh->vring.avail->idx) != 0) { in __vringh_notify_enable()
540 &vrh->vring.avail->idx); in __vringh_notify_enable()
547 return avail == vrh->last_avail_idx; in __vringh_notify_enable()
550 static inline void __vringh_notify_disable(struct vringh *vrh, in __vringh_notify_disable() argument
551 int (*putu16)(const struct vringh *vrh, in __vringh_notify_disable() argument
554 if (!vrh->event_indices) { in __vringh_notify_disable()
556 if (putu16(vrh, &vrh->vring.used->flags, in __vringh_notify_disable()
559 &vrh->vring.used->flags); in __vringh_notify_disable()
565 static inline int getu16_user(const struct vringh *vrh, u16 *val, const __virtio16 *p) in getu16_user() argument
569 *val = vringh16_to_cpu(vrh, v); in getu16_user()
573 static inline int putu16_user(const struct vringh *vrh, __virtio16 *p, u16 val) in putu16_user() argument
575 __virtio16 v = cpu_to_vringh16(vrh, val); in putu16_user()
579 static inline int copydesc_user(const struct vringh *vrh, in copydesc_user() argument
586 static inline int putused_user(const struct vringh *vrh, in putused_user() argument
595 static inline int xfer_from_user(const struct vringh *vrh, void *src, in xfer_from_user() argument
602 static inline int xfer_to_user(const struct vringh *vrh, in xfer_to_user() argument
622 int vringh_init_user(struct vringh *vrh, u64 features, in vringh_init_user() argument
634 vrh->little_endian = (features & (1ULL << VIRTIO_F_VERSION_1)); in vringh_init_user()
635 vrh->event_indices = (features & (1 << VIRTIO_RING_F_EVENT_IDX)); in vringh_init_user()
636 vrh->weak_barriers = weak_barriers; in vringh_init_user()
637 vrh->completed = 0; in vringh_init_user()
638 vrh->last_avail_idx = 0; in vringh_init_user()
639 vrh->last_used_idx = 0; in vringh_init_user()
640 vrh->vring.num = num; in vringh_init_user()
642 vrh->vring.desc = (__force struct vring_desc *)desc; in vringh_init_user()
643 vrh->vring.avail = (__force struct vring_avail *)avail; in vringh_init_user()
644 vrh->vring.used = (__force struct vring_used *)used; in vringh_init_user()
666 int vringh_getdesc_user(struct vringh *vrh, in vringh_getdesc_user() argument
669 bool (*getrange)(struct vringh *vrh, in vringh_getdesc_user() argument
675 *head = vrh->vring.num; in vringh_getdesc_user()
676 err = __vringh_get_head(vrh, getu16_user, &vrh->last_avail_idx); in vringh_getdesc_user()
681 if (err == vrh->vring.num) in vringh_getdesc_user()
705 err = __vringh_iov(vrh, *head, (struct vringh_kiov *)riov, in vringh_getdesc_user()
754 void vringh_abandon_user(struct vringh *vrh, unsigned int num) in vringh_abandon_user() argument
758 vrh->last_avail_idx -= num; in vringh_abandon_user()
771 int vringh_complete_user(struct vringh *vrh, u16 head, u32 len) in vringh_complete_user() argument
775 used.id = cpu_to_vringh32(vrh, head); in vringh_complete_user()
776 used.len = cpu_to_vringh32(vrh, len); in vringh_complete_user()
777 return __vringh_complete(vrh, &used, 1, putu16_user, putused_user); in vringh_complete_user()
790 int vringh_complete_multi_user(struct vringh *vrh, in vringh_complete_multi_user() argument
794 return __vringh_complete(vrh, used, num_used, in vringh_complete_multi_user()
806 bool vringh_notify_enable_user(struct vringh *vrh) in vringh_notify_enable_user() argument
808 return __vringh_notify_enable(vrh, getu16_user, putu16_user); in vringh_notify_enable_user()
819 void vringh_notify_disable_user(struct vringh *vrh) in vringh_notify_disable_user() argument
821 __vringh_notify_disable(vrh, putu16_user); in vringh_notify_disable_user()
831 int vringh_need_notify_user(struct vringh *vrh) in vringh_need_notify_user() argument
833 return __vringh_need_notify(vrh, getu16_user); in vringh_need_notify_user()
838 static inline int getu16_kern(const struct vringh *vrh, in getu16_kern() argument
841 *val = vringh16_to_cpu(vrh, READ_ONCE(*p)); in getu16_kern()
845 static inline int putu16_kern(const struct vringh *vrh, __virtio16 *p, u16 val) in putu16_kern() argument
847 WRITE_ONCE(*p, cpu_to_vringh16(vrh, val)); in putu16_kern()
851 static inline int copydesc_kern(const struct vringh *vrh, in copydesc_kern() argument
858 static inline int putused_kern(const struct vringh *vrh, in putused_kern() argument
867 static inline int xfer_kern(const struct vringh *vrh, void *src, in xfer_kern() argument
874 static inline int kern_xfer(const struct vringh *vrh, void *dst, in kern_xfer() argument
893 int vringh_init_kern(struct vringh *vrh, u64 features, in vringh_init_kern() argument
905 vrh->little_endian = (features & (1ULL << VIRTIO_F_VERSION_1)); in vringh_init_kern()
906 vrh->event_indices = (features & (1 << VIRTIO_RING_F_EVENT_IDX)); in vringh_init_kern()
907 vrh->weak_barriers = weak_barriers; in vringh_init_kern()
908 vrh->completed = 0; in vringh_init_kern()
909 vrh->last_avail_idx = 0; in vringh_init_kern()
910 vrh->last_used_idx = 0; in vringh_init_kern()
911 vrh->vring.num = num; in vringh_init_kern()
912 vrh->vring.desc = desc; in vringh_init_kern()
913 vrh->vring.avail = avail; in vringh_init_kern()
914 vrh->vring.used = used; in vringh_init_kern()
936 int vringh_getdesc_kern(struct vringh *vrh, in vringh_getdesc_kern() argument
944 err = __vringh_get_head(vrh, getu16_kern, &vrh->last_avail_idx); in vringh_getdesc_kern()
949 if (err == vrh->vring.num) in vringh_getdesc_kern()
953 err = __vringh_iov(vrh, *head, riov, wiov, no_range_check, NULL, in vringh_getdesc_kern()
999 void vringh_abandon_kern(struct vringh *vrh, unsigned int num) in vringh_abandon_kern() argument
1003 vrh->last_avail_idx -= num; in vringh_abandon_kern()
1016 int vringh_complete_kern(struct vringh *vrh, u16 head, u32 len) in vringh_complete_kern() argument
1020 used.id = cpu_to_vringh32(vrh, head); in vringh_complete_kern()
1021 used.len = cpu_to_vringh32(vrh, len); in vringh_complete_kern()
1023 return __vringh_complete(vrh, &used, 1, putu16_kern, putused_kern); in vringh_complete_kern()
1034 bool vringh_notify_enable_kern(struct vringh *vrh) in vringh_notify_enable_kern() argument
1036 return __vringh_notify_enable(vrh, getu16_kern, putu16_kern); in vringh_notify_enable_kern()
1047 void vringh_notify_disable_kern(struct vringh *vrh) in vringh_notify_disable_kern() argument
1049 __vringh_notify_disable(vrh, putu16_kern); in vringh_notify_disable_kern()
1059 int vringh_need_notify_kern(struct vringh *vrh) in vringh_need_notify_kern() argument
1061 return __vringh_need_notify(vrh, getu16_kern); in vringh_need_notify_kern()
1067 static int iotlb_translate(const struct vringh *vrh, in iotlb_translate() argument
1072 struct vhost_iotlb *iotlb = vrh->iotlb; in iotlb_translate()
1108 static inline int copy_from_iotlb(const struct vringh *vrh, void *dst, in copy_from_iotlb() argument
1115 ret = iotlb_translate(vrh, (u64)(uintptr_t)src, in copy_from_iotlb()
1127 static inline int copy_to_iotlb(const struct vringh *vrh, void *dst, in copy_to_iotlb() argument
1134 ret = iotlb_translate(vrh, (u64)(uintptr_t)dst, in copy_to_iotlb()
1144 static inline int getu16_iotlb(const struct vringh *vrh, in getu16_iotlb() argument
1152 ret = iotlb_translate(vrh, (u64)(uintptr_t)p, sizeof(*p), in getu16_iotlb()
1159 *val = vringh16_to_cpu(vrh, READ_ONCE(*(__virtio16 *)from)); in getu16_iotlb()
1165 static inline int putu16_iotlb(const struct vringh *vrh, in putu16_iotlb() argument
1173 ret = iotlb_translate(vrh, (u64)(uintptr_t)p, sizeof(*p), in putu16_iotlb()
1180 WRITE_ONCE(*(__virtio16 *)to, cpu_to_vringh16(vrh, val)); in putu16_iotlb()
1186 static inline int copydesc_iotlb(const struct vringh *vrh, in copydesc_iotlb() argument
1191 ret = copy_from_iotlb(vrh, dst, (void *)src, len); in copydesc_iotlb()
1198 static inline int xfer_from_iotlb(const struct vringh *vrh, void *src, in xfer_from_iotlb() argument
1203 ret = copy_from_iotlb(vrh, dst, src, len); in xfer_from_iotlb()
1210 static inline int xfer_to_iotlb(const struct vringh *vrh, in xfer_to_iotlb() argument
1215 ret = copy_to_iotlb(vrh, dst, src, len); in xfer_to_iotlb()
1222 static inline int putused_iotlb(const struct vringh *vrh, in putused_iotlb() argument
1230 ret = copy_to_iotlb(vrh, dst, (void *)src, num * sizeof(*dst)); in putused_iotlb()
1249 int vringh_init_iotlb(struct vringh *vrh, u64 features, in vringh_init_iotlb() argument
1255 return vringh_init_kern(vrh, features, num, weak_barriers, in vringh_init_iotlb()
1265 void vringh_set_iotlb(struct vringh *vrh, struct vhost_iotlb *iotlb) in vringh_set_iotlb() argument
1267 vrh->iotlb = iotlb; in vringh_set_iotlb()
1289 int vringh_getdesc_iotlb(struct vringh *vrh, in vringh_getdesc_iotlb() argument
1297 err = __vringh_get_head(vrh, getu16_iotlb, &vrh->last_avail_idx); in vringh_getdesc_iotlb()
1302 if (err == vrh->vring.num) in vringh_getdesc_iotlb()
1306 err = __vringh_iov(vrh, *head, riov, wiov, no_range_check, NULL, in vringh_getdesc_iotlb()
1324 ssize_t vringh_iov_pull_iotlb(struct vringh *vrh, in vringh_iov_pull_iotlb() argument
1328 return vringh_iov_xfer(vrh, riov, dst, len, xfer_from_iotlb); in vringh_iov_pull_iotlb()
1341 ssize_t vringh_iov_push_iotlb(struct vringh *vrh, in vringh_iov_push_iotlb() argument
1345 return vringh_iov_xfer(vrh, wiov, (void *)src, len, xfer_to_iotlb); in vringh_iov_push_iotlb()
1357 void vringh_abandon_iotlb(struct vringh *vrh, unsigned int num) in vringh_abandon_iotlb() argument
1362 vrh->last_avail_idx -= num; in vringh_abandon_iotlb()
1375 int vringh_complete_iotlb(struct vringh *vrh, u16 head, u32 len) in vringh_complete_iotlb() argument
1379 used.id = cpu_to_vringh32(vrh, head); in vringh_complete_iotlb()
1380 used.len = cpu_to_vringh32(vrh, len); in vringh_complete_iotlb()
1382 return __vringh_complete(vrh, &used, 1, putu16_iotlb, putused_iotlb); in vringh_complete_iotlb()
1393 bool vringh_notify_enable_iotlb(struct vringh *vrh) in vringh_notify_enable_iotlb() argument
1395 return __vringh_notify_enable(vrh, getu16_iotlb, putu16_iotlb); in vringh_notify_enable_iotlb()
1406 void vringh_notify_disable_iotlb(struct vringh *vrh) in vringh_notify_disable_iotlb() argument
1408 __vringh_notify_disable(vrh, putu16_iotlb); in vringh_notify_disable_iotlb()
1418 int vringh_need_notify_iotlb(struct vringh *vrh) in vringh_need_notify_iotlb() argument
1420 return __vringh_need_notify(vrh, getu16_iotlb); in vringh_need_notify_iotlb()