37 /* Returns vring->num if empty, -ve on error. */
46 	err = getu16(vrh, &avail_idx, &vrh->vring.avail->idx);  in __vringh_get_head()
49 			   &vrh->vring.avail->idx);  in __vringh_get_head()
54 		return vrh->vring.num;  in __vringh_get_head()
59 	i = *last_avail_idx & (vrh->vring.num - 1);  in __vringh_get_head()
61 	err = getu16(vrh, &head, &vrh->vring.avail->ring[i]);  in __vringh_get_head()
64 			   *last_avail_idx, &vrh->vring.avail->ring[i]);  in __vringh_get_head()
68 	if (head >= vrh->vring.num) {  in __vringh_get_head()
70 			   head, vrh->vring.num);  in __vringh_get_head()
223 	*descs = vrh->vring.desc;  in return_from_indirect()
224 	*desc_max = vrh->vring.num;  in return_from_indirect()
282 	/* We start traversing vring's descriptor table. */  in __vringh_iov()
283 	descs = vrh->vring.desc;  in __vringh_iov()
284 	desc_max = vrh->vring.num;  in __vringh_iov()
334 		if (count++ == vrh->vring.num) {  in __vringh_iov()
427 	used_ring = vrh->vring.used;  in __vringh_complete()
430 	off = used_idx % vrh->vring.num;  in __vringh_complete()
433 	if (num_used > 1 && unlikely(off + num_used >= vrh->vring.num)) {  in __vringh_complete()
434 		u16 part = vrh->vring.num - off;  in __vringh_complete()
451 	err = putu16(vrh, &vrh->vring.used->idx, used_idx + num_used);  in __vringh_complete()
454 			   &vrh->vring.used->idx);  in __vringh_complete()
480 		err = getu16(vrh, &flags, &vrh->vring.avail->flags);  in __vringh_need_notify()
483 				   &vrh->vring.avail->flags);  in __vringh_need_notify()
490 	err = getu16(vrh, &used_event, &vring_used_event(&vrh->vring));  in __vringh_need_notify()
493 			   &vring_used_event(&vrh->vring));  in __vringh_need_notify()
520 		if (putu16(vrh, &vrh->vring.used->flags, 0) != 0) {  in __vringh_notify_enable()
522 				   &vrh->vring.used->flags);  in __vringh_notify_enable()
526 		if (putu16(vrh, &vring_avail_event(&vrh->vring),  in __vringh_notify_enable()
529 				   &vring_avail_event(&vrh->vring));  in __vringh_notify_enable()
538 	if (getu16(vrh, &avail, &vrh->vring.avail->idx) != 0) {  in __vringh_notify_enable()
540 			   &vrh->vring.avail->idx);  in __vringh_notify_enable()
556 		if (putu16(vrh, &vrh->vring.used->flags,  in __vringh_notify_disable()
559 				   &vrh->vring.used->flags);  in __vringh_notify_disable()
610  * vringh_init_user - initialize a vringh for a userspace vring.
640 	vrh->vring.num = num;  in vringh_init_user()
641 	/* vring expects kernel addresses, but only used via accessors. */  in vringh_init_user()
642 	vrh->vring.desc = (__force struct vring_desc *)desc;  in vringh_init_user()
643 	vrh->vring.avail = (__force struct vring_avail *)avail;  in vringh_init_user()
644 	vrh->vring.used = (__force struct vring_used *)used;  in vringh_init_user()
651  * @vrh: the userspace vring.
661  * *head will be vrh->vring.num.  You may be able to ignore an invalid
675 	*head = vrh->vring.num;  in vringh_getdesc_user()
681 	if (err == vrh->vring.num)  in vringh_getdesc_user()
748  * @vrh: the vring.
764  * @vrh: the vring.
783  * @vrh: the vring.
801  * @vrh: the vring.
804  * now more buffers available in the vring.
814  * @vrh: the vring.
827  * @vrh: the vring we've called vringh_complete_user() on.
882  * vringh_init_kern - initialize a vringh for a kernelspace vring.
911 	vrh->vring.num = num;  in vringh_init_kern()
912 	vrh->vring.desc = desc;  in vringh_init_kern()
913 	vrh->vring.avail = avail;  in vringh_init_kern()
914 	vrh->vring.used = used;  in vringh_init_kern()
921  * @vrh: the kernelspace vring.
931  * *head will be vrh->vring.num.  You may be able to ignore an invalid
949 	if (err == vrh->vring.num)  in vringh_getdesc_kern()
993  * @vrh: the vring.
1009  * @vrh: the vring.
1029  * @vrh: the vring.
1032  * now more buffers available in the vring.
1042  * @vrh: the vring.
1055  * @vrh: the vring we've called vringh_complete_kern() on.
1262  * @vrh: the vring
1263  * @iotlb: iotlb associated with this vring
1274  * @vrh: the kernelspace vring.
1284  * *head will be vrh->vring.num.  You may be able to ignore an invalid
1302 	if (err == vrh->vring.num)  in vringh_getdesc_iotlb()
1317  * @vrh: the vring.
1334  * @vrh: the vring.
1351  * @vrh: the vring.
1368  * @vrh: the vring.
1388  * @vrh: the vring.
1391  * now more buffers available in the vring.
1401  * @vrh: the vring.
1414  * @vrh: the vring we've called vringh_complete_iotlb() on.