38 static inline int __vringh_get_head(const struct vringh *vrh,  in __vringh_get_head()  argument
39 				    int (*getu16)(const struct vringh *vrh,  in __vringh_get_head()  argument
46 	err = getu16(vrh, &avail_idx, &vrh->vring.avail->idx);  in __vringh_get_head()
49 			   &vrh->vring.avail->idx);  in __vringh_get_head()
54 		return vrh->vring.num;  in __vringh_get_head()
57 	virtio_rmb(vrh->weak_barriers);  in __vringh_get_head()
59 	i = *last_avail_idx & (vrh->vring.num - 1);  in __vringh_get_head()
61 	err = getu16(vrh, &head, &vrh->vring.avail->ring[i]);  in __vringh_get_head()
64 			   *last_avail_idx, &vrh->vring.avail->ring[i]);  in __vringh_get_head()
68 	if (head >= vrh->vring.num) {  in __vringh_get_head()
70 			   head, vrh->vring.num);  in __vringh_get_head()
107 static inline ssize_t vringh_iov_xfer(struct vringh *vrh,  in vringh_iov_xfer()  argument
110 				      int (*xfer)(const struct vringh *vrh,  in vringh_iov_xfer()  argument
120 		err = xfer(vrh, iov->iov[iov->i].iov_base, ptr, partlen);  in vringh_iov_xfer()
133 static inline bool range_check(struct vringh *vrh, u64 addr, size_t *len,  in range_check()  argument
139 		if (!getrange(vrh, addr, range))  in range_check()
167 static inline bool no_range_check(struct vringh *vrh, u64 addr, size_t *len,  in no_range_check()  argument
176 static int move_to_indirect(const struct vringh *vrh,  in move_to_indirect()  argument
189 	len = vringh32_to_cpu(vrh, desc->len);  in move_to_indirect()
196 	if (desc->flags & cpu_to_vringh16(vrh, VRING_DESC_F_NEXT))  in move_to_indirect()
197 		*up_next = vringh16_to_cpu(vrh, desc->next);  in move_to_indirect()
235 static u16 __cold return_from_indirect(const struct vringh *vrh, int *up_next,  in return_from_indirect()  argument
241 	*descs = vrh->vring.desc;  in return_from_indirect()
242 	*desc_max = vrh->vring.num;  in return_from_indirect()
246 static int slow_copy(struct vringh *vrh, void *dst, const void *src,  in slow_copy()  argument
247 		     bool (*rcheck)(struct vringh *vrh, u64 addr, size_t *len,  in slow_copy()  argument
249 				    bool (*getrange)(struct vringh *vrh,  in slow_copy()  argument
252 		     bool (*getrange)(struct vringh *vrh,  in slow_copy()
256 		     int (*copy)(const struct vringh *vrh,  in slow_copy()  argument
268 		if (!rcheck(vrh, addr, &part, range, getrange))  in slow_copy()
271 		err = copy(vrh, dst, src, part);  in slow_copy()
283 __vringh_iov(struct vringh *vrh, u16 i,  in __vringh_iov()  argument
286 	     bool (*rcheck)(struct vringh *vrh, u64 addr, size_t *len,  in __vringh_iov()  argument
292 	     int (*copy)(const struct vringh *vrh,  in __vringh_iov()  argument
301 	descs = vrh->vring.desc;  in __vringh_iov()
302 	desc_max = vrh->vring.num;  in __vringh_iov()
320 			err = slow_copy(vrh, &desc, &descs[i], rcheck, getrange,  in __vringh_iov()
323 			err = copy(vrh, &desc, &descs[i], sizeof(desc));  in __vringh_iov()
328 			     cpu_to_vringh16(vrh, VRING_DESC_F_INDIRECT))) {  in __vringh_iov()
329 			u64 a = vringh64_to_cpu(vrh, desc.addr);  in __vringh_iov()
332 			len = vringh32_to_cpu(vrh, desc.len);  in __vringh_iov()
333 			if (!rcheck(vrh, a, &len, &range, getrange)) {  in __vringh_iov()
338 			if (unlikely(len != vringh32_to_cpu(vrh, desc.len))) {  in __vringh_iov()
345 			err = move_to_indirect(vrh, &up_next, &i, addr, &desc,  in __vringh_iov()
357 		if (count > vrh->vring.num || indirect_count > desc_max) {  in __vringh_iov()
363 		if (desc.flags & cpu_to_vringh16(vrh, VRING_DESC_F_WRITE))  in __vringh_iov()
384 		len = vringh32_to_cpu(vrh, desc.len);  in __vringh_iov()
385 		if (!rcheck(vrh, vringh64_to_cpu(vrh, desc.addr), &len, &range,  in __vringh_iov()
390 		addr = (void *)(unsigned long)(vringh64_to_cpu(vrh, desc.addr) +  in __vringh_iov()
403 		if (unlikely(len != vringh32_to_cpu(vrh, desc.len))) {  in __vringh_iov()
404 			desc.len = cpu_to_vringh32(vrh,  in __vringh_iov()
405 				   vringh32_to_cpu(vrh, desc.len) - len);  in __vringh_iov()
406 			desc.addr = cpu_to_vringh64(vrh,  in __vringh_iov()
407 				    vringh64_to_cpu(vrh, desc.addr) + len);  in __vringh_iov()
411 		if (desc.flags & cpu_to_vringh16(vrh, VRING_DESC_F_NEXT)) {  in __vringh_iov()
412 			i = vringh16_to_cpu(vrh, desc.next);  in __vringh_iov()
416 				i = return_from_indirect(vrh, &up_next,  in __vringh_iov()
437 static inline int __vringh_complete(struct vringh *vrh,  in __vringh_complete()  argument
440 				    int (*putu16)(const struct vringh *vrh,  in __vringh_complete()  argument
442 				    int (*putused)(const struct vringh *vrh,  in __vringh_complete()  argument
451 	used_ring = vrh->vring.used;  in __vringh_complete()
452 	used_idx = vrh->last_used_idx + vrh->completed;  in __vringh_complete()
454 	off = used_idx % vrh->vring.num;  in __vringh_complete()
457 	if (num_used > 1 && unlikely(off + num_used >= vrh->vring.num)) {  in __vringh_complete()
458 		u16 part = vrh->vring.num - off;  in __vringh_complete()
459 		err = putused(vrh, &used_ring->ring[off], used, part);  in __vringh_complete()
461 			err = putused(vrh, &used_ring->ring[0], used + part,  in __vringh_complete()
464 		err = putused(vrh, &used_ring->ring[off], used, num_used);  in __vringh_complete()
473 	virtio_wmb(vrh->weak_barriers);  in __vringh_complete()
475 	err = putu16(vrh, &vrh->vring.used->idx, used_idx + num_used);  in __vringh_complete()
478 			   &vrh->vring.used->idx);  in __vringh_complete()
482 	vrh->completed += num_used;  in __vringh_complete()
487 static inline int __vringh_need_notify(struct vringh *vrh,  in __vringh_need_notify()  argument
488 				       int (*getu16)(const struct vringh *vrh,  in __vringh_need_notify()  argument
499 	virtio_mb(vrh->weak_barriers);  in __vringh_need_notify()
502 	if (!vrh->event_indices) {  in __vringh_need_notify()
504 		err = getu16(vrh, &flags, &vrh->vring.avail->flags);  in __vringh_need_notify()
507 				   &vrh->vring.avail->flags);  in __vringh_need_notify()
514 	err = getu16(vrh, &used_event, &vring_used_event(&vrh->vring));  in __vringh_need_notify()
517 			   &vring_used_event(&vrh->vring));  in __vringh_need_notify()
522 	if (unlikely(vrh->completed > 0xffff))  in __vringh_need_notify()
526 					  vrh->last_used_idx + vrh->completed,  in __vringh_need_notify()
527 					  vrh->last_used_idx);  in __vringh_need_notify()
529 	vrh->last_used_idx += vrh->completed;  in __vringh_need_notify()
530 	vrh->completed = 0;  in __vringh_need_notify()
534 static inline bool __vringh_notify_enable(struct vringh *vrh,  in __vringh_notify_enable()  argument
535 					  int (*getu16)(const struct vringh *vrh,  in __vringh_notify_enable()  argument
537 					  int (*putu16)(const struct vringh *vrh,  in __vringh_notify_enable()  argument
542 	if (!vrh->event_indices) {  in __vringh_notify_enable()
544 		if (putu16(vrh, &vrh->vring.used->flags, 0) != 0) {  in __vringh_notify_enable()
546 				   &vrh->vring.used->flags);  in __vringh_notify_enable()
550 		if (putu16(vrh, &vring_avail_event(&vrh->vring),  in __vringh_notify_enable()
551 			   vrh->last_avail_idx) != 0) {  in __vringh_notify_enable()
553 				   &vring_avail_event(&vrh->vring));  in __vringh_notify_enable()
560 	virtio_mb(vrh->weak_barriers);  in __vringh_notify_enable()
562 	if (getu16(vrh, &avail, &vrh->vring.avail->idx) != 0) {  in __vringh_notify_enable()
564 			   &vrh->vring.avail->idx);  in __vringh_notify_enable()
571 	return avail == vrh->last_avail_idx;  in __vringh_notify_enable()
574 static inline void __vringh_notify_disable(struct vringh *vrh,  in __vringh_notify_disable()  argument
575 					   int (*putu16)(const struct vringh *vrh,  in __vringh_notify_disable()  argument
578 	if (!vrh->event_indices) {  in __vringh_notify_disable()
580 		if (putu16(vrh, &vrh->vring.used->flags,  in __vringh_notify_disable()
583 				   &vrh->vring.used->flags);  in __vringh_notify_disable()
589 static inline int getu16_user(const struct vringh *vrh, u16 *val, const __virtio16 *p)  in getu16_user()  argument
593 	*val = vringh16_to_cpu(vrh, v);  in getu16_user()
597 static inline int putu16_user(const struct vringh *vrh, __virtio16 *p, u16 val)  in putu16_user()  argument
599 	__virtio16 v = cpu_to_vringh16(vrh, val);  in putu16_user()
603 static inline int copydesc_user(const struct vringh *vrh,  in copydesc_user()  argument
610 static inline int putused_user(const struct vringh *vrh,  in putused_user()  argument
619 static inline int xfer_from_user(const struct vringh *vrh, void *src,  in xfer_from_user()  argument
626 static inline int xfer_to_user(const struct vringh *vrh,  in xfer_to_user()  argument
646 int vringh_init_user(struct vringh *vrh, u64 features,  in vringh_init_user()  argument
658 	vrh->little_endian = (features & (1ULL << VIRTIO_F_VERSION_1));  in vringh_init_user()
659 	vrh->event_indices = (features & (1 << VIRTIO_RING_F_EVENT_IDX));  in vringh_init_user()
660 	vrh->weak_barriers = weak_barriers;  in vringh_init_user()
661 	vrh->completed = 0;  in vringh_init_user()
662 	vrh->last_avail_idx = 0;  in vringh_init_user()
663 	vrh->last_used_idx = 0;  in vringh_init_user()
664 	vrh->vring.num = num;  in vringh_init_user()
666 	vrh->vring.desc = (__force struct vring_desc *)desc;  in vringh_init_user()
667 	vrh->vring.avail = (__force struct vring_avail *)avail;  in vringh_init_user()
668 	vrh->vring.used = (__force struct vring_used *)used;  in vringh_init_user()
693 int vringh_getdesc_user(struct vringh *vrh,  in vringh_getdesc_user()  argument
696 			bool (*getrange)(struct vringh *vrh,  in vringh_getdesc_user()  argument
702 	*head = vrh->vring.num;  in vringh_getdesc_user()
703 	err = __vringh_get_head(vrh, getu16_user, &vrh->last_avail_idx);  in vringh_getdesc_user()
708 	if (err == vrh->vring.num)  in vringh_getdesc_user()
732 	err = __vringh_iov(vrh, *head, (struct vringh_kiov *)riov,  in vringh_getdesc_user()
781 void vringh_abandon_user(struct vringh *vrh, unsigned int num)  in vringh_abandon_user()  argument
785 	vrh->last_avail_idx -= num;  in vringh_abandon_user()
798 int vringh_complete_user(struct vringh *vrh, u16 head, u32 len)  in vringh_complete_user()  argument
802 	used.id = cpu_to_vringh32(vrh, head);  in vringh_complete_user()
803 	used.len = cpu_to_vringh32(vrh, len);  in vringh_complete_user()
804 	return __vringh_complete(vrh, &used, 1, putu16_user, putused_user);  in vringh_complete_user()
817 int vringh_complete_multi_user(struct vringh *vrh,  in vringh_complete_multi_user()  argument
821 	return __vringh_complete(vrh, used, num_used,  in vringh_complete_multi_user()
833 bool vringh_notify_enable_user(struct vringh *vrh)  in vringh_notify_enable_user()  argument
835 	return __vringh_notify_enable(vrh, getu16_user, putu16_user);  in vringh_notify_enable_user()
846 void vringh_notify_disable_user(struct vringh *vrh)  in vringh_notify_disable_user()  argument
848 	__vringh_notify_disable(vrh, putu16_user);  in vringh_notify_disable_user()
858 int vringh_need_notify_user(struct vringh *vrh)  in vringh_need_notify_user()  argument
860 	return __vringh_need_notify(vrh, getu16_user);  in vringh_need_notify_user()
865 static inline int getu16_kern(const struct vringh *vrh,  in getu16_kern()  argument
868 	*val = vringh16_to_cpu(vrh, READ_ONCE(*p));  in getu16_kern()
872 static inline int putu16_kern(const struct vringh *vrh, __virtio16 *p, u16 val)  in putu16_kern()  argument
874 	WRITE_ONCE(*p, cpu_to_vringh16(vrh, val));  in putu16_kern()
878 static inline int copydesc_kern(const struct vringh *vrh,  in copydesc_kern()  argument
885 static inline int putused_kern(const struct vringh *vrh,  in putused_kern()  argument
894 static inline int xfer_kern(const struct vringh *vrh, void *src,  in xfer_kern()  argument
901 static inline int kern_xfer(const struct vringh *vrh, void *dst,  in kern_xfer()  argument
920 int vringh_init_kern(struct vringh *vrh, u64 features,  in vringh_init_kern()  argument
932 	vrh->little_endian = (features & (1ULL << VIRTIO_F_VERSION_1));  in vringh_init_kern()
933 	vrh->event_indices = (features & (1 << VIRTIO_RING_F_EVENT_IDX));  in vringh_init_kern()
934 	vrh->weak_barriers = weak_barriers;  in vringh_init_kern()
935 	vrh->completed = 0;  in vringh_init_kern()
936 	vrh->last_avail_idx = 0;  in vringh_init_kern()
937 	vrh->last_used_idx = 0;  in vringh_init_kern()
938 	vrh->vring.num = num;  in vringh_init_kern()
939 	vrh->vring.desc = desc;  in vringh_init_kern()
940 	vrh->vring.avail = avail;  in vringh_init_kern()
941 	vrh->vring.used = used;  in vringh_init_kern()
966 int vringh_getdesc_kern(struct vringh *vrh,  in vringh_getdesc_kern()  argument
974 	err = __vringh_get_head(vrh, getu16_kern, &vrh->last_avail_idx);  in vringh_getdesc_kern()
979 	if (err == vrh->vring.num)  in vringh_getdesc_kern()
983 	err = __vringh_iov(vrh, *head, riov, wiov, no_range_check, NULL,  in vringh_getdesc_kern()
1029 void vringh_abandon_kern(struct vringh *vrh, unsigned int num)  in vringh_abandon_kern()  argument
1033 	vrh->last_avail_idx -= num;  in vringh_abandon_kern()
1046 int vringh_complete_kern(struct vringh *vrh, u16 head, u32 len)  in vringh_complete_kern()  argument
1050 	used.id = cpu_to_vringh32(vrh, head);  in vringh_complete_kern()
1051 	used.len = cpu_to_vringh32(vrh, len);  in vringh_complete_kern()
1053 	return __vringh_complete(vrh, &used, 1, putu16_kern, putused_kern);  in vringh_complete_kern()
1064 bool vringh_notify_enable_kern(struct vringh *vrh)  in vringh_notify_enable_kern()  argument
1066 	return __vringh_notify_enable(vrh, getu16_kern, putu16_kern);  in vringh_notify_enable_kern()
1077 void vringh_notify_disable_kern(struct vringh *vrh)  in vringh_notify_disable_kern()  argument
1079 	__vringh_notify_disable(vrh, putu16_kern);  in vringh_notify_disable_kern()
1089 int vringh_need_notify_kern(struct vringh *vrh)  in vringh_need_notify_kern()  argument
1091 	return __vringh_need_notify(vrh, getu16_kern);  in vringh_need_notify_kern()
1097 static int iotlb_translate(const struct vringh *vrh,  in iotlb_translate()  argument
1103 	struct vhost_iotlb *iotlb = vrh->iotlb;  in iotlb_translate()
1107 	spin_lock(vrh->iotlb_lock);  in iotlb_translate()
1138 	spin_unlock(vrh->iotlb_lock);  in iotlb_translate()
1146 static inline int copy_from_iotlb(const struct vringh *vrh, void *dst,  in copy_from_iotlb()  argument
1157 		ret = iotlb_translate(vrh, (u64)(uintptr_t)src,  in copy_from_iotlb()
1179 static inline int copy_to_iotlb(const struct vringh *vrh, void *dst,  in copy_to_iotlb()  argument
1190 		ret = iotlb_translate(vrh, (u64)(uintptr_t)dst,  in copy_to_iotlb()
1212 static inline int getu16_iotlb(const struct vringh *vrh,  in getu16_iotlb()  argument
1220 	ret = iotlb_translate(vrh, (u64)(uintptr_t)p, sizeof(*p), NULL,  in getu16_iotlb()
1227 	*val = vringh16_to_cpu(vrh, READ_ONCE(*(__virtio16 *)from));  in getu16_iotlb()
1233 static inline int putu16_iotlb(const struct vringh *vrh,  in putu16_iotlb()  argument
1241 	ret = iotlb_translate(vrh, (u64)(uintptr_t)p, sizeof(*p), NULL,  in putu16_iotlb()
1248 	WRITE_ONCE(*(__virtio16 *)to, cpu_to_vringh16(vrh, val));  in putu16_iotlb()
1254 static inline int copydesc_iotlb(const struct vringh *vrh,  in copydesc_iotlb()  argument
1259 	ret = copy_from_iotlb(vrh, dst, (void *)src, len);  in copydesc_iotlb()
1266 static inline int xfer_from_iotlb(const struct vringh *vrh, void *src,  in xfer_from_iotlb()  argument
1271 	ret = copy_from_iotlb(vrh, dst, src, len);  in xfer_from_iotlb()
1278 static inline int xfer_to_iotlb(const struct vringh *vrh,  in xfer_to_iotlb()  argument
1283 	ret = copy_to_iotlb(vrh, dst, src, len);  in xfer_to_iotlb()
1290 static inline int putused_iotlb(const struct vringh *vrh,  in putused_iotlb()  argument
1298 	ret = copy_to_iotlb(vrh, dst, (void *)src, num * sizeof(*dst));  in putused_iotlb()
1317 int vringh_init_iotlb(struct vringh *vrh, u64 features,  in vringh_init_iotlb()  argument
1323 	return vringh_init_kern(vrh, features, num, weak_barriers,  in vringh_init_iotlb()
1334 void vringh_set_iotlb(struct vringh *vrh, struct vhost_iotlb *iotlb,  in vringh_set_iotlb()  argument
1337 	vrh->iotlb = iotlb;  in vringh_set_iotlb()
1338 	vrh->iotlb_lock = iotlb_lock;  in vringh_set_iotlb()
1363 int vringh_getdesc_iotlb(struct vringh *vrh,  in vringh_getdesc_iotlb()  argument
1371 	err = __vringh_get_head(vrh, getu16_iotlb, &vrh->last_avail_idx);  in vringh_getdesc_iotlb()
1376 	if (err == vrh->vring.num)  in vringh_getdesc_iotlb()
1380 	err = __vringh_iov(vrh, *head, riov, wiov, no_range_check, NULL,  in vringh_getdesc_iotlb()
1398 ssize_t vringh_iov_pull_iotlb(struct vringh *vrh,  in vringh_iov_pull_iotlb()  argument
1402 	return vringh_iov_xfer(vrh, riov, dst, len, xfer_from_iotlb);  in vringh_iov_pull_iotlb()
1415 ssize_t vringh_iov_push_iotlb(struct vringh *vrh,  in vringh_iov_push_iotlb()  argument
1419 	return vringh_iov_xfer(vrh, wiov, (void *)src, len, xfer_to_iotlb);  in vringh_iov_push_iotlb()
1431 void vringh_abandon_iotlb(struct vringh *vrh, unsigned int num)  in vringh_abandon_iotlb()  argument
1436 	vrh->last_avail_idx -= num;  in vringh_abandon_iotlb()
1449 int vringh_complete_iotlb(struct vringh *vrh, u16 head, u32 len)  in vringh_complete_iotlb()  argument
1453 	used.id = cpu_to_vringh32(vrh, head);  in vringh_complete_iotlb()
1454 	used.len = cpu_to_vringh32(vrh, len);  in vringh_complete_iotlb()
1456 	return __vringh_complete(vrh, &used, 1, putu16_iotlb, putused_iotlb);  in vringh_complete_iotlb()
1467 bool vringh_notify_enable_iotlb(struct vringh *vrh)  in vringh_notify_enable_iotlb()  argument
1469 	return __vringh_notify_enable(vrh, getu16_iotlb, putu16_iotlb);  in vringh_notify_enable_iotlb()
1480 void vringh_notify_disable_iotlb(struct vringh *vrh)  in vringh_notify_disable_iotlb()  argument
1482 	__vringh_notify_disable(vrh, putu16_iotlb);  in vringh_notify_disable_iotlb()
1492 int vringh_need_notify_iotlb(struct vringh *vrh)  in vringh_need_notify_iotlb()  argument
1494 	return __vringh_need_notify(vrh, getu16_iotlb);  in vringh_need_notify_iotlb()