124 			struct vring vring;  member
152 			} vring;  member
321  * making all of the arch DMA ops work on the vring device itself
457 	struct vring_virtqueue *vring = to_vvq(vq);  in virtqueue_add_desc_split()  local
458 	struct vring_desc_extra *extra = vring->split.desc_extra;  in virtqueue_add_desc_split()
514 		WARN_ON_ONCE(total_sg > vq->split.vring.num && !vq->indirect);  in virtqueue_add_split()
525 		desc = vq->split.vring.desc;  in virtqueue_add_split()
579 		vq->split.desc_extra[prev & (vq->split.vring.num - 1)].flags &=  in virtqueue_add_split()
590 		virtqueue_add_desc_split(_vq, vq->split.vring.desc,  in virtqueue_add_split()
615 	avail = vq->split.avail_idx_shadow & (vq->split.vring.num - 1);  in virtqueue_add_split()
616 	vq->split.vring.avail->ring[avail] = cpu_to_virtio16(_vq->vdev, head);  in virtqueue_add_split()
622 	vq->split.vring.avail->idx = cpu_to_virtio16(_vq->vdev,  in virtqueue_add_split()
681 					vring_avail_event(&vq->split.vring)),  in virtqueue_kick_prepare_split()
684 		needs_kick = !(vq->split.vring.used->flags &  in virtqueue_kick_prepare_split()
704 	while (vq->split.vring.desc[i].flags & nextflag) {  in detach_buf_split()
745 			vq->split.vring.used->idx);  in more_used_split()
773 	last_used = (vq->last_used_idx & (vq->split.vring.num - 1));  in virtqueue_get_buf_ctx_split()
775 			vq->split.vring.used->ring[last_used].id);  in virtqueue_get_buf_ctx_split()
777 			vq->split.vring.used->ring[last_used].len);  in virtqueue_get_buf_ctx_split()
779 	if (unlikely(i >= vq->split.vring.num)) {  in virtqueue_get_buf_ctx_split()
797 				&vring_used_event(&vq->split.vring),  in virtqueue_get_buf_ctx_split()
814 			vring_used_event(&vq->split.vring) = 0x0;  in virtqueue_disable_cb_split()
816 			vq->split.vring.avail->flags =  in virtqueue_disable_cb_split()
837 			vq->split.vring.avail->flags =  in virtqueue_enable_cb_prepare_split()
841 	vring_used_event(&vq->split.vring) = cpu_to_virtio16(_vq->vdev,  in virtqueue_enable_cb_prepare_split()
852 			vq->split.vring.used->idx);  in virtqueue_poll_split()
870 			vq->split.vring.avail->flags =  in virtqueue_enable_cb_delayed_split()
878 			&vring_used_event(&vq->split.vring),  in virtqueue_enable_cb_delayed_split()
881 	if (unlikely((u16)(virtio16_to_cpu(_vq->vdev, vq->split.vring.used->idx)  in virtqueue_enable_cb_delayed_split()
899 	for (i = 0; i < vq->split.vring.num; i++) {  in virtqueue_detach_unused_buf_split()
906 		vq->split.vring.avail->idx = cpu_to_virtio16(_vq->vdev,  in virtqueue_detach_unused_buf_split()
912 	BUG_ON(vq->vq.num_free != vq->split.vring.num);  in virtqueue_detach_unused_buf_split()
934 	struct vring vring;  in vring_create_virtqueue_split()  local
965 	vring_init(&vring, num, queue, vring_align);  in vring_create_virtqueue_split()
967 	vq = __vring_new_virtqueue(index, vring, vdev, weak_barriers, context,  in vring_create_virtqueue_split()
1078 	BUG_ON(id == vq->packed.vring.num);  in virtqueue_add_indirect_packed()
1102 	vq->packed.vring.desc[head].addr = cpu_to_le64(addr);  in virtqueue_add_indirect_packed()
1103 	vq->packed.vring.desc[head].len = cpu_to_le32(total_sg *  in virtqueue_add_indirect_packed()
1105 	vq->packed.vring.desc[head].id = cpu_to_le16(id);  in virtqueue_add_indirect_packed()
1121 	vq->packed.vring.desc[head].flags = cpu_to_le16(VRING_DESC_F_INDIRECT |  in virtqueue_add_indirect_packed()
1129 	if (n >= vq->packed.vring.num) {  in virtqueue_add_indirect_packed()
1201 	WARN_ON_ONCE(total_sg > vq->packed.vring.num && !vq->indirect);  in virtqueue_add_packed()
1203 	desc = vq->packed.vring.desc;  in virtqueue_add_packed()
1215 	BUG_ON(id == vq->packed.vring.num);  in virtqueue_add_packed()
1247 			if ((unlikely(++i >= vq->packed.vring.num))) {  in virtqueue_add_packed()
1278 	vq->packed.vring.desc[head].flags = head_flags;  in virtqueue_add_packed()
1300 		if (i >= vq->packed.vring.num)  in virtqueue_add_packed()
1333 	snapshot.u32 = *(u32 *)vq->packed.vring.device;  in virtqueue_kick_prepare_packed()
1349 		event_idx -= vq->packed.vring.num;  in virtqueue_kick_prepare_packed()
1409 	flags = le16_to_cpu(vq->packed.vring.desc[idx].flags);  in is_used_desc_packed()
1447 	id = le16_to_cpu(vq->packed.vring.desc[last_used].id);  in virtqueue_get_buf_ctx_packed()
1448 	*len = le32_to_cpu(vq->packed.vring.desc[last_used].len);  in virtqueue_get_buf_ctx_packed()
1450 	if (unlikely(id >= vq->packed.vring.num)) {  in virtqueue_get_buf_ctx_packed()
1464 	if (unlikely(vq->last_used_idx >= vq->packed.vring.num)) {  in virtqueue_get_buf_ctx_packed()
1465 		vq->last_used_idx -= vq->packed.vring.num;  in virtqueue_get_buf_ctx_packed()
1476 				&vq->packed.vring.driver->off_wrap,  in virtqueue_get_buf_ctx_packed()
1493 		vq->packed.vring.driver->flags =  in virtqueue_disable_cb_packed()
1510 		vq->packed.vring.driver->off_wrap =  in virtqueue_enable_cb_prepare_packed()
1525 		vq->packed.vring.driver->flags =  in virtqueue_enable_cb_prepare_packed()
1561 		bufs = (vq->packed.vring.num - vq->vq.num_free) * 3 / 4;  in virtqueue_enable_cb_delayed_packed()
1565 		if (used_idx >= vq->packed.vring.num) {  in virtqueue_enable_cb_delayed_packed()
1566 			used_idx -= vq->packed.vring.num;  in virtqueue_enable_cb_delayed_packed()
1570 		vq->packed.vring.driver->off_wrap = cpu_to_le16(used_idx |  in virtqueue_enable_cb_delayed_packed()
1584 		vq->packed.vring.driver->flags =  in virtqueue_enable_cb_delayed_packed()
1613 	for (i = 0; i < vq->packed.vring.num; i++) {  in virtqueue_detach_unused_buf_packed()
1623 	BUG_ON(vq->vq.num_free != vq->packed.vring.num);  in virtqueue_detach_unused_buf_packed()
1725 	vq->packed.vring.num = num;  in vring_create_virtqueue_packed()
1726 	vq->packed.vring.desc = ring;  in vring_create_virtqueue_packed()
1727 	vq->packed.vring.driver = driver;  in vring_create_virtqueue_packed()
1728 	vq->packed.vring.device = device;  in vring_create_virtqueue_packed()
1755 		vq->packed.vring.driver->flags =  in vring_create_virtqueue_packed()
2163 					struct vring vring,  in __vring_new_virtqueue()  argument
2184 	vq->vq.num_free = vring.num;  in __vring_new_virtqueue()
2209 	vq->split.vring = vring;  in __vring_new_virtqueue()
2217 			vq->split.vring.avail->flags = cpu_to_virtio16(vdev,  in __vring_new_virtqueue()
2221 	vq->split.desc_state = kmalloc_array(vring.num,  in __vring_new_virtqueue()
2226 	vq->split.desc_extra = vring_alloc_desc_extra(vq, vring.num);  in __vring_new_virtqueue()
2232 	memset(vq->split.desc_state, 0, vring.num *  in __vring_new_virtqueue()
2284 	struct vring vring;  in vring_new_virtqueue()  local
2289 	vring_init(&vring, num, pages, vring_align);  in vring_new_virtqueue()
2290 	return __vring_new_virtqueue(index, vring, vdev, weak_barriers, context,  in vring_new_virtqueue()
2307 					 vq->packed.vring.desc,  in vring_del_virtqueue()
2312 					 vq->packed.vring.driver,  in vring_del_virtqueue()
2317 					 vq->packed.vring.device,  in vring_del_virtqueue()
2325 					 vq->split.vring.desc,  in vring_del_virtqueue()
2365  * virtqueue_get_vring_size - return the size of the virtqueue's vring
2366  * @_vq: the struct virtqueue containing the vring of interest.
2368  * Returns the size of the vring.  This is mainly used for boasting to
2376 	return vq->packed_ring ? vq->packed.vring.num : vq->split.vring.num;  in virtqueue_get_vring_size()
2430 		((char *)vq->split.vring.avail - (char *)vq->split.vring.desc);  in virtqueue_get_avail_addr()
2444 		((char *)vq->split.vring.used - (char *)vq->split.vring.desc);  in virtqueue_get_used_addr()
2449 const struct vring *virtqueue_get_vring(struct virtqueue *vq)  in virtqueue_get_vring()
2451 	return &to_vvq(vq)->split.vring;  in virtqueue_get_vring()