67 	struct vring vring;  member
308 		WARN_ON_ONCE(total_sg > vq->vring.num && !vq->indirect);  in virtqueue_add()
319 		desc = vq->vring.desc;  in virtqueue_add()
375 		vq->vring.desc[head].flags = cpu_to_virtio16(_vq->vdev, VRING_DESC_F_INDIRECT);  in virtqueue_add()
376 		vq->vring.desc[head].addr = cpu_to_virtio64(_vq->vdev, addr);  in virtqueue_add()
378 		vq->vring.desc[head].len = cpu_to_virtio32(_vq->vdev, total_sg * sizeof(struct vring_desc));  in virtqueue_add()
386 		vq->free_head = virtio16_to_cpu(_vq->vdev, vq->vring.desc[head].next);  in virtqueue_add()
399 	avail = vq->avail_idx_shadow & (vq->vring.num - 1);  in virtqueue_add()
400 	vq->vring.avail->ring[avail] = cpu_to_virtio16(_vq->vdev, head);  in virtqueue_add()
406 	vq->vring.avail->idx = cpu_to_virtio16(_vq->vdev, vq->avail_idx_shadow);  in virtqueue_add()
427 		i = virtio16_to_cpu(_vq->vdev, vq->vring.desc[i].next);  in virtqueue_add()
574 		needs_kick = vring_need_event(virtio16_to_cpu(_vq->vdev, vring_avail_event(&vq->vring)),  in virtqueue_kick_prepare()
577 		needs_kick = !(vq->vring.used->flags & cpu_to_virtio16(_vq->vdev, VRING_USED_F_NO_NOTIFY));  in virtqueue_kick_prepare()
640 	while (vq->vring.desc[i].flags & nextflag) {  in detach_buf()
641 		vring_unmap_one(vq, &vq->vring.desc[i]);  in detach_buf()
642 		i = virtio16_to_cpu(vq->vq.vdev, vq->vring.desc[i].next);  in detach_buf()
646 	vring_unmap_one(vq, &vq->vring.desc[i]);  in detach_buf()
647 	vq->vring.desc[i].next = cpu_to_virtio16(vq->vq.vdev, vq->free_head);  in detach_buf()
661 		len = virtio32_to_cpu(vq->vq.vdev, vq->vring.desc[head].len);  in detach_buf()
663 		BUG_ON(!(vq->vring.desc[head].flags &  in detach_buf()
679 	return vq->last_used_idx != virtio16_to_cpu(vq->vq.vdev, vq->vring.used->idx);  in more_used()
722 	last_used = (vq->last_used_idx & (vq->vring.num - 1));  in virtqueue_get_buf_ctx()
723 	i = virtio32_to_cpu(_vq->vdev, vq->vring.used->ring[last_used].id);  in virtqueue_get_buf_ctx()
724 	*len = virtio32_to_cpu(_vq->vdev, vq->vring.used->ring[last_used].len);  in virtqueue_get_buf_ctx()
726 	if (unlikely(i >= vq->vring.num)) {  in virtqueue_get_buf_ctx()
744 				&vring_used_event(&vq->vring),  in virtqueue_get_buf_ctx()
777 			vq->vring.avail->flags = cpu_to_virtio16(_vq->vdev, vq->avail_flags_shadow);  in virtqueue_disable_cb()
810 			vq->vring.avail->flags = cpu_to_virtio16(_vq->vdev, vq->avail_flags_shadow);  in virtqueue_enable_cb_prepare()
812 	vring_used_event(&vq->vring) = cpu_to_virtio16(_vq->vdev, last_used_idx = vq->last_used_idx);  in virtqueue_enable_cb_prepare()
832 	return (u16)last_used_idx != virtio16_to_cpu(_vq->vdev, vq->vring.used->idx);  in virtqueue_poll()
882 			vq->vring.avail->flags = cpu_to_virtio16(_vq->vdev, vq->avail_flags_shadow);  in virtqueue_enable_cb_delayed()
888 			&vring_used_event(&vq->vring),  in virtqueue_enable_cb_delayed()
891 	if (unlikely((u16)(virtio16_to_cpu(_vq->vdev, vq->vring.used->idx) - vq->last_used_idx) > bufs)) {  in virtqueue_enable_cb_delayed()
917 	for (i = 0; i < vq->vring.num; i++) {  in virtqueue_detach_unused_buf()
924 		vq->vring.avail->idx = cpu_to_virtio16(_vq->vdev, vq->avail_idx_shadow);  in virtqueue_detach_unused_buf()
929 	BUG_ON(vq->vq.num_free != vq->vring.num);  in virtqueue_detach_unused_buf()
957 					struct vring vring,  in __vring_new_virtqueue()  argument
968 	vq = kmalloc(sizeof(*vq) + vring.num * sizeof(struct vring_desc_state),  in __vring_new_virtqueue()
973 	vq->vring = vring;  in __vring_new_virtqueue()
977 	vq->vq.num_free = vring.num;  in __vring_new_virtqueue()
1003 			vq->vring.avail->flags = cpu_to_virtio16(vdev, vq->avail_flags_shadow);  in __vring_new_virtqueue()
1008 	for (i = 0; i < vring.num-1; i++)  in __vring_new_virtqueue()
1009 		vq->vring.desc[i].next = cpu_to_virtio16(vdev, i + 1);  in __vring_new_virtqueue()
1010 	memset(vq->desc_state, 0, vring.num * sizeof(struct vring_desc_state));  in __vring_new_virtqueue()
1074 	struct vring vring;  in vring_create_virtqueue()  local
1103 	vring_init(&vring, num, queue, vring_align);  in vring_create_virtqueue()
1105 	vq = __vring_new_virtqueue(index, vring, vdev, weak_barriers, context,  in vring_create_virtqueue()
1132 	struct vring vring;  in vring_new_virtqueue()  local
1133 	vring_init(&vring, num, pages, vring_align);  in vring_new_virtqueue()
1134 	return __vring_new_virtqueue(index, vring, vdev, weak_barriers, context,  in vring_new_virtqueue()
1145 				 vq->vring.desc, vq->queue_dma_addr);  in vring_del_virtqueue()
1187 	return vq->vring.num;  in virtqueue_get_vring_size()
1231 		((char *)vq->vring.avail - (char *)vq->vring.desc);  in virtqueue_get_avail_addr()
1242 		((char *)vq->vring.used - (char *)vq->vring.desc);  in virtqueue_get_used_addr()
1246 const struct vring *virtqueue_get_vring(struct virtqueue *vq)  in virtqueue_get_vring()
1248 	return &to_vvq(vq)->vring;  in virtqueue_get_vring()