37 /* Returns vring->num if empty, -ve on error. */
46 	err = getu16(vrh, &avail_idx, &vrh->vring.avail->idx);  in __vringh_get_head()
49 			   &vrh->vring.avail->idx);  in __vringh_get_head()
54 		return vrh->vring.num;  in __vringh_get_head()
59 	i = *last_avail_idx & (vrh->vring.num - 1);  in __vringh_get_head()
61 	err = getu16(vrh, &head, &vrh->vring.avail->ring[i]);  in __vringh_get_head()
64 			   *last_avail_idx, &vrh->vring.avail->ring[i]);  in __vringh_get_head()
68 	if (head >= vrh->vring.num) {  in __vringh_get_head()
70 			   head, vrh->vring.num);  in __vringh_get_head()
241 	*descs = vrh->vring.desc;  in return_from_indirect()
242 	*desc_max = vrh->vring.num;  in return_from_indirect()
300 	/* We start traversing vring's descriptor table. */  in __vringh_iov()
301 	descs = vrh->vring.desc;  in __vringh_iov()
302 	desc_max = vrh->vring.num;  in __vringh_iov()
357 		if (count > vrh->vring.num || indirect_count > desc_max) {  in __vringh_iov()
451 	used_ring = vrh->vring.used;  in __vringh_complete()
454 	off = used_idx % vrh->vring.num;  in __vringh_complete()
457 	if (num_used > 1 && unlikely(off + num_used >= vrh->vring.num)) {  in __vringh_complete()
458 		u16 part = vrh->vring.num - off;  in __vringh_complete()
475 	err = putu16(vrh, &vrh->vring.used->idx, used_idx + num_used);  in __vringh_complete()
478 			   &vrh->vring.used->idx);  in __vringh_complete()
504 		err = getu16(vrh, &flags, &vrh->vring.avail->flags);  in __vringh_need_notify()
507 				   &vrh->vring.avail->flags);  in __vringh_need_notify()
514 	err = getu16(vrh, &used_event, &vring_used_event(&vrh->vring));  in __vringh_need_notify()
517 			   &vring_used_event(&vrh->vring));  in __vringh_need_notify()
544 		if (putu16(vrh, &vrh->vring.used->flags, 0) != 0) {  in __vringh_notify_enable()
546 				   &vrh->vring.used->flags);  in __vringh_notify_enable()
550 		if (putu16(vrh, &vring_avail_event(&vrh->vring),  in __vringh_notify_enable()
553 				   &vring_avail_event(&vrh->vring));  in __vringh_notify_enable()
562 	if (getu16(vrh, &avail, &vrh->vring.avail->idx) != 0) {  in __vringh_notify_enable()
564 			   &vrh->vring.avail->idx);  in __vringh_notify_enable()
580 		if (putu16(vrh, &vrh->vring.used->flags,  in __vringh_notify_disable()
583 				   &vrh->vring.used->flags);  in __vringh_notify_disable()
634  * vringh_init_user - initialize a vringh for a userspace vring.
664 	vrh->vring.num = num;  in vringh_init_user()
665 	/* vring expects kernel addresses, but only used via accessors. */  in vringh_init_user()
666 	vrh->vring.desc = (__force struct vring_desc *)desc;  in vringh_init_user()
667 	vrh->vring.avail = (__force struct vring_avail *)avail;  in vringh_init_user()
668 	vrh->vring.used = (__force struct vring_used *)used;  in vringh_init_user()
675  * @vrh: the userspace vring.
685  * *head will be vrh->vring.num.  You may be able to ignore an invalid
702 	*head = vrh->vring.num;  in vringh_getdesc_user()
708 	if (err == vrh->vring.num)  in vringh_getdesc_user()
775  * @vrh: the vring.
791  * @vrh: the vring.
810  * @vrh: the vring.
828  * @vrh: the vring.
831  * now more buffers available in the vring.
841  * @vrh: the vring.
854  * @vrh: the vring we've called vringh_complete_user() on.
909  * vringh_init_kern - initialize a vringh for a kernelspace vring.
938 	vrh->vring.num = num;  in vringh_init_kern()
939 	vrh->vring.desc = desc;  in vringh_init_kern()
940 	vrh->vring.avail = avail;  in vringh_init_kern()
941 	vrh->vring.used = used;  in vringh_init_kern()
948  * @vrh: the kernelspace vring.
958  * *head will be vrh->vring.num.  You may be able to ignore an invalid
979 	if (err == vrh->vring.num)  in vringh_getdesc_kern()
1023  * @vrh: the vring.
1039  * @vrh: the vring.
1059  * @vrh: the vring.
1062  * now more buffers available in the vring.
1072  * @vrh: the vring.
1085  * @vrh: the vring we've called vringh_complete_kern() on.
1330  * @vrh: the vring
1331  * @iotlb: iotlb associated with this vring
1345  * @vrh: the kernelspace vring.
1355  * *head will be vrh->vring.num.  You may be able to ignore an invalid
1376 	if (err == vrh->vring.num)  in vringh_getdesc_iotlb()
1391  * @vrh: the vring.
1408  * @vrh: the vring.
1425  * @vrh: the vring.
1442  * @vrh: the vring.
1462  * @vrh: the vring.
1465  * now more buffers available in the vring.
1475  * @vrh: the vring.
1488  * @vrh: the vring we've called vringh_complete_iotlb() on.