46 	err = getu16(vrh, &avail_idx, &vrh->vring.avail->idx);  in __vringh_get_head()
49 			   &vrh->vring.avail->idx);  in __vringh_get_head()
54 		return vrh->vring.num;  in __vringh_get_head()
59 	i = *last_avail_idx & (vrh->vring.num - 1);  in __vringh_get_head()
61 	err = getu16(vrh, &head, &vrh->vring.avail->ring[i]);  in __vringh_get_head()
64 			   *last_avail_idx, &vrh->vring.avail->ring[i]);  in __vringh_get_head()
68 	if (head >= vrh->vring.num) {  in __vringh_get_head()
70 			   head, vrh->vring.num);  in __vringh_get_head()
241 	*descs = vrh->vring.desc;  in return_from_indirect()
242 	*desc_max = vrh->vring.num;  in return_from_indirect()
301 	descs = vrh->vring.desc;  in __vringh_iov()
302 	desc_max = vrh->vring.num;  in __vringh_iov()
352 		if (count++ == vrh->vring.num) {  in __vringh_iov()
445 	used_ring = vrh->vring.used;  in __vringh_complete()
448 	off = used_idx % vrh->vring.num;  in __vringh_complete()
451 	if (num_used > 1 && unlikely(off + num_used >= vrh->vring.num)) {  in __vringh_complete()
452 		u16 part = vrh->vring.num - off;  in __vringh_complete()
469 	err = putu16(vrh, &vrh->vring.used->idx, used_idx + num_used);  in __vringh_complete()
472 			   &vrh->vring.used->idx);  in __vringh_complete()
498 		err = getu16(vrh, &flags, &vrh->vring.avail->flags);  in __vringh_need_notify()
501 				   &vrh->vring.avail->flags);  in __vringh_need_notify()
508 	err = getu16(vrh, &used_event, &vring_used_event(&vrh->vring));  in __vringh_need_notify()
511 			   &vring_used_event(&vrh->vring));  in __vringh_need_notify()
538 		if (putu16(vrh, &vrh->vring.used->flags, 0) != 0) {  in __vringh_notify_enable()
540 				   &vrh->vring.used->flags);  in __vringh_notify_enable()
544 		if (putu16(vrh, &vring_avail_event(&vrh->vring),  in __vringh_notify_enable()
547 				   &vring_avail_event(&vrh->vring));  in __vringh_notify_enable()
556 	if (getu16(vrh, &avail, &vrh->vring.avail->idx) != 0) {  in __vringh_notify_enable()
558 			   &vrh->vring.avail->idx);  in __vringh_notify_enable()
574 		if (putu16(vrh, &vrh->vring.used->flags,  in __vringh_notify_disable()
577 				   &vrh->vring.used->flags);  in __vringh_notify_disable()
658 	vrh->vring.num = num;  in vringh_init_user()
660 	vrh->vring.desc = (__force struct vring_desc *)desc;  in vringh_init_user()
661 	vrh->vring.avail = (__force struct vring_avail *)avail;  in vringh_init_user()
662 	vrh->vring.used = (__force struct vring_used *)used;  in vringh_init_user()
696 	*head = vrh->vring.num;  in vringh_getdesc_user()
702 	if (err == vrh->vring.num)  in vringh_getdesc_user()
932 	vrh->vring.num = num;  in vringh_init_kern()
933 	vrh->vring.desc = desc;  in vringh_init_kern()
934 	vrh->vring.avail = avail;  in vringh_init_kern()
935 	vrh->vring.used = used;  in vringh_init_kern()
973 	if (err == vrh->vring.num)  in vringh_getdesc_kern()
1336 	if (err == vrh->vring.num)  in vringh_getdesc_iotlb()