91 	struct vring vring;  member
112 	 * vring.
125 	} vring;  member
343  * making all of the arch DMA ops work on the vring device itself
496 	struct vring_virtqueue *vring = to_vvq(vq);  in virtqueue_add_desc_split()  local
497 	struct vring_desc_extra *extra = vring->split.desc_extra;  in virtqueue_add_desc_split()
553 		WARN_ON_ONCE(total_sg > vq->split.vring.num && !vq->indirect);  in virtqueue_add_split()
564 		desc = vq->split.vring.desc;  in virtqueue_add_split()
618 		vq->split.desc_extra[prev & (vq->split.vring.num - 1)].flags &=  in virtqueue_add_split()
629 		virtqueue_add_desc_split(_vq, vq->split.vring.desc,  in virtqueue_add_split()
654 	avail = vq->split.avail_idx_shadow & (vq->split.vring.num - 1);  in virtqueue_add_split()
655 	vq->split.vring.avail->ring[avail] = cpu_to_virtio16(_vq->vdev, head);  in virtqueue_add_split()
661 	vq->split.vring.avail->idx = cpu_to_virtio16(_vq->vdev,  in virtqueue_add_split()
720 					vring_avail_event(&vq->split.vring)),  in virtqueue_kick_prepare_split()
723 		needs_kick = !(vq->split.vring.used->flags &  in virtqueue_kick_prepare_split()
743 	while (vq->split.vring.desc[i].flags & nextflag) {  in detach_buf_split()
784 			vq->split.vring.used->idx);  in more_used_split()
812 	last_used = (vq->last_used_idx & (vq->split.vring.num - 1));  in virtqueue_get_buf_ctx_split()
814 			vq->split.vring.used->ring[last_used].id);  in virtqueue_get_buf_ctx_split()
816 			vq->split.vring.used->ring[last_used].len);  in virtqueue_get_buf_ctx_split()
818 	if (unlikely(i >= vq->split.vring.num)) {  in virtqueue_get_buf_ctx_split()
836 				&vring_used_event(&vq->split.vring),  in virtqueue_get_buf_ctx_split()
853 			vring_used_event(&vq->split.vring) = 0x0;  in virtqueue_disable_cb_split()
855 			vq->split.vring.avail->flags =  in virtqueue_disable_cb_split()
876 			vq->split.vring.avail->flags =  in virtqueue_enable_cb_prepare_split()
880 	vring_used_event(&vq->split.vring) = cpu_to_virtio16(_vq->vdev,  in virtqueue_enable_cb_prepare_split()
891 			vq->split.vring.used->idx);  in virtqueue_poll_split()
909 			vq->split.vring.avail->flags =  in virtqueue_enable_cb_delayed_split()
917 			&vring_used_event(&vq->split.vring),  in virtqueue_enable_cb_delayed_split()
920 	if (unlikely((u16)(virtio16_to_cpu(_vq->vdev, vq->split.vring.used->idx)  in virtqueue_enable_cb_delayed_split()
938 	for (i = 0; i < vq->split.vring.num; i++) {  in virtqueue_detach_unused_buf_split()
945 		vq->split.vring.avail->idx = cpu_to_virtio16(_vq->vdev,  in virtqueue_detach_unused_buf_split()
951 	BUG_ON(vq->vq.num_free != vq->split.vring.num);  in virtqueue_detach_unused_buf_split()
971 			vring_split->vring.avail->flags = cpu_to_virtio16(vdev,  in virtqueue_vring_init_split()
980 	num = vq->split.vring.num;  in virtqueue_reinit_split()
982 	vq->split.vring.avail->flags = 0;  in virtqueue_reinit_split()
983 	vq->split.vring.avail->idx = 0;  in virtqueue_reinit_split()
986 	vq->split.vring.avail->ring[num] = 0;  in virtqueue_reinit_split()
988 	vq->split.vring.used->flags = 0;  in virtqueue_reinit_split()
989 	vq->split.vring.used->idx = 0;  in virtqueue_reinit_split()
992 	*(__virtio16 *)&(vq->split.vring.used->ring[num]) = 0;  in virtqueue_reinit_split()
1012 	u32 num = vring_split->vring.num;  in vring_alloc_state_extra_split()
1038 			 vring_split->vring.desc,  in vring_free_split()
1082 	vring_init(&vring_split->vring, num, queue, vring_align);  in vring_alloc_queue_split()
1147 	virtqueue_init(vq, vring_split.vring.num);  in virtqueue_resize_split()
1258 	BUG_ON(id == vq->packed.vring.num);  in virtqueue_add_indirect_packed()
1282 	vq->packed.vring.desc[head].addr = cpu_to_le64(addr);  in virtqueue_add_indirect_packed()
1283 	vq->packed.vring.desc[head].len = cpu_to_le32(total_sg *  in virtqueue_add_indirect_packed()
1285 	vq->packed.vring.desc[head].id = cpu_to_le16(id);  in virtqueue_add_indirect_packed()
1301 	vq->packed.vring.desc[head].flags = cpu_to_le16(VRING_DESC_F_INDIRECT |  in virtqueue_add_indirect_packed()
1309 	if (n >= vq->packed.vring.num) {  in virtqueue_add_indirect_packed()
1389 	WARN_ON_ONCE(total_sg > vq->packed.vring.num && !vq->indirect);  in virtqueue_add_packed()
1391 	desc = vq->packed.vring.desc;  in virtqueue_add_packed()
1403 	BUG_ON(id == vq->packed.vring.num);  in virtqueue_add_packed()
1435 			if ((unlikely(++i >= vq->packed.vring.num))) {  in virtqueue_add_packed()
1466 	vq->packed.vring.desc[head].flags = head_flags;  in virtqueue_add_packed()
1487 		if (i >= vq->packed.vring.num)  in virtqueue_add_packed()
1520 	snapshot.u32 = *(u32 *)vq->packed.vring.device;  in virtqueue_kick_prepare_packed()
1536 		event_idx -= vq->packed.vring.num;  in virtqueue_kick_prepare_packed()
1596 	flags = le16_to_cpu(vq->packed.vring.desc[idx].flags);  in is_used_desc_packed()
1643 	id = le16_to_cpu(vq->packed.vring.desc[last_used].id);  in virtqueue_get_buf_ctx_packed()
1644 	*len = le32_to_cpu(vq->packed.vring.desc[last_used].len);  in virtqueue_get_buf_ctx_packed()
1646 	if (unlikely(id >= vq->packed.vring.num)) {  in virtqueue_get_buf_ctx_packed()
1660 	if (unlikely(last_used >= vq->packed.vring.num)) {  in virtqueue_get_buf_ctx_packed()
1661 		last_used -= vq->packed.vring.num;  in virtqueue_get_buf_ctx_packed()
1675 				&vq->packed.vring.driver->off_wrap,  in virtqueue_get_buf_ctx_packed()
1690 		vq->packed.vring.driver->flags =  in virtqueue_disable_cb_packed()
1707 		vq->packed.vring.driver->off_wrap =  in virtqueue_enable_cb_prepare_packed()
1720 		vq->packed.vring.driver->flags =  in virtqueue_enable_cb_prepare_packed()
1755 		bufs = (vq->packed.vring.num - vq->vq.num_free) * 3 / 4;  in virtqueue_enable_cb_delayed_packed()
1760 		if (used_idx >= vq->packed.vring.num) {  in virtqueue_enable_cb_delayed_packed()
1761 			used_idx -= vq->packed.vring.num;  in virtqueue_enable_cb_delayed_packed()
1765 		vq->packed.vring.driver->off_wrap = cpu_to_le16(used_idx |  in virtqueue_enable_cb_delayed_packed()
1779 		vq->packed.vring.driver->flags =  in virtqueue_enable_cb_delayed_packed()
1809 	for (i = 0; i < vq->packed.vring.num; i++) {  in virtqueue_detach_unused_buf_packed()
1819 	BUG_ON(vq->vq.num_free != vq->packed.vring.num);  in virtqueue_detach_unused_buf_packed()
1846 	if (vring_packed->vring.desc)  in vring_free_packed()
1848 				 vring_packed->vring.desc,  in vring_free_packed()
1851 	if (vring_packed->vring.driver)  in vring_free_packed()
1853 				 vring_packed->vring.driver,  in vring_free_packed()
1856 	if (vring_packed->vring.device)  in vring_free_packed()
1858 				 vring_packed->vring.device,  in vring_free_packed()
1882 	vring_packed->vring.desc         = ring;  in vring_alloc_queue_packed()
1894 	vring_packed->vring.driver          = driver;  in vring_alloc_queue_packed()
1904 	vring_packed->vring.device          = device;  in vring_alloc_queue_packed()
1907 	vring_packed->vring.num = num;  in vring_alloc_queue_packed()
1920 	u32 num = vring_packed->vring.num;  in vring_alloc_state_extra_packed()
1954 		vring_packed->vring.driver->flags =  in virtqueue_vring_init_packed()
1970 	memset(vq->packed.vring.device, 0, vq->packed.event_size_in_bytes);  in virtqueue_reinit_packed()
1971 	memset(vq->packed.vring.driver, 0, vq->packed.event_size_in_bytes);  in virtqueue_reinit_packed()
1974 	memset(vq->packed.vring.desc, 0, vq->packed.ring_size_in_bytes);  in virtqueue_reinit_packed()
1976 	virtqueue_init(vq, vq->packed.vring.num);  in virtqueue_reinit_packed()
2066 	virtqueue_init(vq, vring_packed.vring.num);  in virtqueue_resize_packed()
2457 			      "virtio vring IRQ raised before DRIVER_OK");  in vring_interrupt()
2527 	virtqueue_init(vq, vring_split->vring.num);  in __vring_new_virtqueue()
2561  * virtqueue_resize - resize the vring of vq
2566  * When it is really necessary to create a new vring, it will set the current vq
2568  * that is no longer used. Only after the new vring is successfully created, the
2569  * old vring will be released.
2601 	if ((vq->packed_ring ? vq->packed.vring.num : vq->split.vring.num) == num)  in virtqueue_resize()
2646 	vring_init(&vring_split.vring, num, pages, vring_align);  in vring_new_virtqueue()
2660 					 vq->packed.vring.desc,  in vring_free()
2665 					 vq->packed.vring.driver,  in vring_free()
2670 					 vq->packed.vring.device,  in vring_free()
2678 					 vq->split.vring.desc,  in vring_free()
2730  * virtqueue_get_vring_size - return the size of the virtqueue's vring
2731  * @_vq: the struct virtqueue containing the vring of interest.
2733  * Returns the size of the vring.  This is mainly used for boasting to
2741 	return vq->packed_ring ? vq->packed.vring.num : vq->split.vring.num;  in virtqueue_get_vring_size()
2841 		((char *)vq->split.vring.avail - (char *)vq->split.vring.desc);  in virtqueue_get_avail_addr()
2855 		((char *)vq->split.vring.used - (char *)vq->split.vring.desc);  in virtqueue_get_used_addr()
2860 const struct vring *virtqueue_get_vring(struct virtqueue *vq)  in virtqueue_get_vring()
2862 	return &to_vvq(vq)->split.vring;  in virtqueue_get_vring()