37 /* Returns vring->num if empty, -ve on error. */
46 	err = getu16(vrh, &avail_idx, &vrh->vring.avail->idx);  in __vringh_get_head()
49 			   &vrh->vring.avail->idx);  in __vringh_get_head()
54 		return vrh->vring.num;  in __vringh_get_head()
59 	i = *last_avail_idx & (vrh->vring.num - 1);  in __vringh_get_head()
61 	err = getu16(vrh, &head, &vrh->vring.avail->ring[i]);  in __vringh_get_head()
64 			   *last_avail_idx, &vrh->vring.avail->ring[i]);  in __vringh_get_head()
68 	if (head >= vrh->vring.num) {  in __vringh_get_head()
70 			   head, vrh->vring.num);  in __vringh_get_head()
241 	*descs = vrh->vring.desc;  in return_from_indirect()
242 	*desc_max = vrh->vring.num;  in return_from_indirect()
300 	/* We start traversing vring's descriptor table. */  in __vringh_iov()
301 	descs = vrh->vring.desc;  in __vringh_iov()
302 	desc_max = vrh->vring.num;  in __vringh_iov()
352 		if (count++ == vrh->vring.num) {  in __vringh_iov()
445 	used_ring = vrh->vring.used;  in __vringh_complete()
448 	off = used_idx % vrh->vring.num;  in __vringh_complete()
451 	if (num_used > 1 && unlikely(off + num_used >= vrh->vring.num)) {  in __vringh_complete()
452 		u16 part = vrh->vring.num - off;  in __vringh_complete()
469 	err = putu16(vrh, &vrh->vring.used->idx, used_idx + num_used);  in __vringh_complete()
472 			   &vrh->vring.used->idx);  in __vringh_complete()
498 		err = getu16(vrh, &flags, &vrh->vring.avail->flags);  in __vringh_need_notify()
501 				   &vrh->vring.avail->flags);  in __vringh_need_notify()
508 	err = getu16(vrh, &used_event, &vring_used_event(&vrh->vring));  in __vringh_need_notify()
511 			   &vring_used_event(&vrh->vring));  in __vringh_need_notify()
538 		if (putu16(vrh, &vrh->vring.used->flags, 0) != 0) {  in __vringh_notify_enable()
540 				   &vrh->vring.used->flags);  in __vringh_notify_enable()
544 		if (putu16(vrh, &vring_avail_event(&vrh->vring),  in __vringh_notify_enable()
547 				   &vring_avail_event(&vrh->vring));  in __vringh_notify_enable()
556 	if (getu16(vrh, &avail, &vrh->vring.avail->idx) != 0) {  in __vringh_notify_enable()
558 			   &vrh->vring.avail->idx);  in __vringh_notify_enable()
574 		if (putu16(vrh, &vrh->vring.used->flags,  in __vringh_notify_disable()
577 				   &vrh->vring.used->flags);  in __vringh_notify_disable()
628  * vringh_init_user - initialize a vringh for a userspace vring.
658 	vrh->vring.num = num;  in vringh_init_user()
659 	/* vring expects kernel addresses, but only used via accessors. */  in vringh_init_user()
660 	vrh->vring.desc = (__force struct vring_desc *)desc;  in vringh_init_user()
661 	vrh->vring.avail = (__force struct vring_avail *)avail;  in vringh_init_user()
662 	vrh->vring.used = (__force struct vring_used *)used;  in vringh_init_user()
669  * @vrh: the userspace vring.
679  * *head will be vrh->vring.num.  You may be able to ignore an invalid
696 	*head = vrh->vring.num;  in vringh_getdesc_user()
702 	if (err == vrh->vring.num)  in vringh_getdesc_user()
769  * @vrh: the vring.
785  * @vrh: the vring.
804  * @vrh: the vring.
822  * @vrh: the vring.
825  * now more buffers available in the vring.
835  * @vrh: the vring.
848  * @vrh: the vring we've called vringh_complete_user() on.
903  * vringh_init_kern - initialize a vringh for a kernelspace vring.
932 	vrh->vring.num = num;  in vringh_init_kern()
933 	vrh->vring.desc = desc;  in vringh_init_kern()
934 	vrh->vring.avail = avail;  in vringh_init_kern()
935 	vrh->vring.used = used;  in vringh_init_kern()
942  * @vrh: the kernelspace vring.
952  * *head will be vrh->vring.num.  You may be able to ignore an invalid
973 	if (err == vrh->vring.num)  in vringh_getdesc_kern()
1017  * @vrh: the vring.
1033  * @vrh: the vring.
1053  * @vrh: the vring.
1056  * now more buffers available in the vring.
1066  * @vrh: the vring.
1079  * @vrh: the vring we've called vringh_complete_kern() on.
1290  * @vrh: the vring
1291  * @iotlb: iotlb associated with this vring
1305  * @vrh: the kernelspace vring.
1315  * *head will be vrh->vring.num.  You may be able to ignore an invalid
1336 	if (err == vrh->vring.num)  in vringh_getdesc_iotlb()
1351  * @vrh: the vring.
1368  * @vrh: the vring.
1385  * @vrh: the vring.
1402  * @vrh: the vring.
1422  * @vrh: the vring.
1425  * now more buffers available in the vring.
1435  * @vrh: the vring.
1448  * @vrh: the vring we've called vringh_complete_iotlb() on.