91 	/* Is this a packed ring? */
144 		/* Available for packed ring */
182 		} packed;  member
984  * Packed ring specific functions - *_packed().
1066 	head = vq->packed.next_avail_idx;  in virtqueue_add_indirect_packed()
1078 	BUG_ON(id == vq->packed.vring.num);  in virtqueue_add_indirect_packed()
1102 	vq->packed.vring.desc[head].addr = cpu_to_le64(addr);  in virtqueue_add_indirect_packed()
1103 	vq->packed.vring.desc[head].len = cpu_to_le32(total_sg *  in virtqueue_add_indirect_packed()
1105 	vq->packed.vring.desc[head].id = cpu_to_le16(id);  in virtqueue_add_indirect_packed()
1108 		vq->packed.desc_extra[id].addr = addr;  in virtqueue_add_indirect_packed()
1109 		vq->packed.desc_extra[id].len = total_sg *  in virtqueue_add_indirect_packed()
1111 		vq->packed.desc_extra[id].flags = VRING_DESC_F_INDIRECT |  in virtqueue_add_indirect_packed()
1112 						  vq->packed.avail_used_flags;  in virtqueue_add_indirect_packed()
1121 	vq->packed.vring.desc[head].flags = cpu_to_le16(VRING_DESC_F_INDIRECT |  in virtqueue_add_indirect_packed()
1122 						vq->packed.avail_used_flags);  in virtqueue_add_indirect_packed()
1129 	if (n >= vq->packed.vring.num) {  in virtqueue_add_indirect_packed()
1131 		vq->packed.avail_wrap_counter ^= 1;  in virtqueue_add_indirect_packed()
1132 		vq->packed.avail_used_flags ^=  in virtqueue_add_indirect_packed()
1136 	vq->packed.next_avail_idx = n;  in virtqueue_add_indirect_packed()
1137 	vq->free_head = vq->packed.desc_extra[id].next;  in virtqueue_add_indirect_packed()
1140 	vq->packed.desc_state[id].num = 1;  in virtqueue_add_indirect_packed()
1141 	vq->packed.desc_state[id].data = data;  in virtqueue_add_indirect_packed()
1142 	vq->packed.desc_state[id].indir_desc = desc;  in virtqueue_add_indirect_packed()
1143 	vq->packed.desc_state[id].last = id;  in virtqueue_add_indirect_packed()
1198 	head = vq->packed.next_avail_idx;  in virtqueue_add_packed()
1199 	avail_used_flags = vq->packed.avail_used_flags;  in virtqueue_add_packed()
1201 	WARN_ON_ONCE(total_sg > vq->packed.vring.num && !vq->indirect);  in virtqueue_add_packed()
1203 	desc = vq->packed.vring.desc;  in virtqueue_add_packed()
1215 	BUG_ON(id == vq->packed.vring.num);  in virtqueue_add_packed()
1226 			flags = cpu_to_le16(vq->packed.avail_used_flags |  in virtqueue_add_packed()
1239 				vq->packed.desc_extra[curr].addr = addr;  in virtqueue_add_packed()
1240 				vq->packed.desc_extra[curr].len = sg->length;  in virtqueue_add_packed()
1241 				vq->packed.desc_extra[curr].flags =  in virtqueue_add_packed()
1245 			curr = vq->packed.desc_extra[curr].next;  in virtqueue_add_packed()
1247 			if ((unlikely(++i >= vq->packed.vring.num))) {  in virtqueue_add_packed()
1249 				vq->packed.avail_used_flags ^=  in virtqueue_add_packed()
1257 		vq->packed.avail_wrap_counter ^= 1;  in virtqueue_add_packed()
1263 	vq->packed.next_avail_idx = i;  in virtqueue_add_packed()
1267 	vq->packed.desc_state[id].num = descs_used;  in virtqueue_add_packed()
1268 	vq->packed.desc_state[id].data = data;  in virtqueue_add_packed()
1269 	vq->packed.desc_state[id].indir_desc = ctx;  in virtqueue_add_packed()
1270 	vq->packed.desc_state[id].last = prev;  in virtqueue_add_packed()
1278 	vq->packed.vring.desc[head].flags = head_flags;  in virtqueue_add_packed()
1291 	vq->packed.avail_used_flags = avail_used_flags;  in virtqueue_add_packed()
1297 					 &vq->packed.desc_extra[curr]);  in virtqueue_add_packed()
1298 		curr = vq->packed.desc_extra[curr].next;  in virtqueue_add_packed()
1300 		if (i >= vq->packed.vring.num)  in virtqueue_add_packed()
1329 	old = vq->packed.next_avail_idx - vq->num_added;  in virtqueue_kick_prepare_packed()
1330 	new = vq->packed.next_avail_idx;  in virtqueue_kick_prepare_packed()
1333 	snapshot.u32 = *(u32 *)vq->packed.vring.device;  in virtqueue_kick_prepare_packed()
1348 	if (wrap_counter != vq->packed.avail_wrap_counter)  in virtqueue_kick_prepare_packed()
1349 		event_idx -= vq->packed.vring.num;  in virtqueue_kick_prepare_packed()
1364 	state = &vq->packed.desc_state[id];  in detach_buf_packed()
1369 	vq->packed.desc_extra[state->last].next = vq->free_head;  in detach_buf_packed()
1377 				&vq->packed.desc_extra[curr]);  in detach_buf_packed()
1378 			curr = vq->packed.desc_extra[curr].next;  in detach_buf_packed()
1391 			len = vq->packed.desc_extra[id].len;  in detach_buf_packed()
1409 	flags = le16_to_cpu(vq->packed.vring.desc[idx].flags);  in is_used_desc_packed()
1419 			vq->packed.used_wrap_counter);  in more_used_packed()
1447 	id = le16_to_cpu(vq->packed.vring.desc[last_used].id);  in virtqueue_get_buf_ctx_packed()
1448 	*len = le32_to_cpu(vq->packed.vring.desc[last_used].len);  in virtqueue_get_buf_ctx_packed()
1450 	if (unlikely(id >= vq->packed.vring.num)) {  in virtqueue_get_buf_ctx_packed()
1454 	if (unlikely(!vq->packed.desc_state[id].data)) {  in virtqueue_get_buf_ctx_packed()
1460 	ret = vq->packed.desc_state[id].data;  in virtqueue_get_buf_ctx_packed()
1463 	vq->last_used_idx += vq->packed.desc_state[id].num;  in virtqueue_get_buf_ctx_packed()
1464 	if (unlikely(vq->last_used_idx >= vq->packed.vring.num)) {  in virtqueue_get_buf_ctx_packed()
1465 		vq->last_used_idx -= vq->packed.vring.num;  in virtqueue_get_buf_ctx_packed()
1466 		vq->packed.used_wrap_counter ^= 1;  in virtqueue_get_buf_ctx_packed()
1474 	if (vq->packed.event_flags_shadow == VRING_PACKED_EVENT_FLAG_DESC)  in virtqueue_get_buf_ctx_packed()
1476 				&vq->packed.vring.driver->off_wrap,  in virtqueue_get_buf_ctx_packed()
1478 					(vq->packed.used_wrap_counter <<  in virtqueue_get_buf_ctx_packed()
1491 	if (vq->packed.event_flags_shadow != VRING_PACKED_EVENT_FLAG_DISABLE) {  in virtqueue_disable_cb_packed()
1492 		vq->packed.event_flags_shadow = VRING_PACKED_EVENT_FLAG_DISABLE;  in virtqueue_disable_cb_packed()
1493 		vq->packed.vring.driver->flags =  in virtqueue_disable_cb_packed()
1494 			cpu_to_le16(vq->packed.event_flags_shadow);  in virtqueue_disable_cb_packed()
1510 		vq->packed.vring.driver->off_wrap =  in virtqueue_enable_cb_prepare_packed()
1512 				(vq->packed.used_wrap_counter <<  in virtqueue_enable_cb_prepare_packed()
1521 	if (vq->packed.event_flags_shadow == VRING_PACKED_EVENT_FLAG_DISABLE) {  in virtqueue_enable_cb_prepare_packed()
1522 		vq->packed.event_flags_shadow = vq->event ?  in virtqueue_enable_cb_prepare_packed()
1525 		vq->packed.vring.driver->flags =  in virtqueue_enable_cb_prepare_packed()
1526 				cpu_to_le16(vq->packed.event_flags_shadow);  in virtqueue_enable_cb_prepare_packed()
1530 	return vq->last_used_idx | ((u16)vq->packed.used_wrap_counter <<  in virtqueue_enable_cb_prepare_packed()
1561 		bufs = (vq->packed.vring.num - vq->vq.num_free) * 3 / 4;  in virtqueue_enable_cb_delayed_packed()
1562 		wrap_counter = vq->packed.used_wrap_counter;  in virtqueue_enable_cb_delayed_packed()
1565 		if (used_idx >= vq->packed.vring.num) {  in virtqueue_enable_cb_delayed_packed()
1566 			used_idx -= vq->packed.vring.num;  in virtqueue_enable_cb_delayed_packed()
1570 		vq->packed.vring.driver->off_wrap = cpu_to_le16(used_idx |  in virtqueue_enable_cb_delayed_packed()
1580 	if (vq->packed.event_flags_shadow == VRING_PACKED_EVENT_FLAG_DISABLE) {  in virtqueue_enable_cb_delayed_packed()
1581 		vq->packed.event_flags_shadow = vq->event ?  in virtqueue_enable_cb_delayed_packed()
1584 		vq->packed.vring.driver->flags =  in virtqueue_enable_cb_delayed_packed()
1585 				cpu_to_le16(vq->packed.event_flags_shadow);  in virtqueue_enable_cb_delayed_packed()
1596 				vq->packed.used_wrap_counter)) {  in virtqueue_enable_cb_delayed_packed()
1613 	for (i = 0; i < vq->packed.vring.num; i++) {  in virtqueue_detach_unused_buf_packed()
1614 		if (!vq->packed.desc_state[i].data)  in virtqueue_detach_unused_buf_packed()
1617 		buf = vq->packed.desc_state[i].data;  in virtqueue_detach_unused_buf_packed()
1623 	BUG_ON(vq->vq.num_free != vq->packed.vring.num);  in virtqueue_detach_unused_buf_packed()
1718 	vq->packed.ring_dma_addr = ring_dma_addr;  in vring_create_virtqueue_packed()
1719 	vq->packed.driver_event_dma_addr = driver_event_dma_addr;  in vring_create_virtqueue_packed()
1720 	vq->packed.device_event_dma_addr = device_event_dma_addr;  in vring_create_virtqueue_packed()
1722 	vq->packed.ring_size_in_bytes = ring_size_in_bytes;  in vring_create_virtqueue_packed()
1723 	vq->packed.event_size_in_bytes = event_size_in_bytes;  in vring_create_virtqueue_packed()
1725 	vq->packed.vring.num = num;  in vring_create_virtqueue_packed()
1726 	vq->packed.vring.desc = ring;  in vring_create_virtqueue_packed()
1727 	vq->packed.vring.driver = driver;  in vring_create_virtqueue_packed()
1728 	vq->packed.vring.device = device;  in vring_create_virtqueue_packed()
1730 	vq->packed.next_avail_idx = 0;  in vring_create_virtqueue_packed()
1731 	vq->packed.avail_wrap_counter = 1;  in vring_create_virtqueue_packed()
1732 	vq->packed.used_wrap_counter = 1;  in vring_create_virtqueue_packed()
1733 	vq->packed.event_flags_shadow = 0;  in vring_create_virtqueue_packed()
1734 	vq->packed.avail_used_flags = 1 << VRING_PACKED_DESC_F_AVAIL;  in vring_create_virtqueue_packed()
1736 	vq->packed.desc_state = kmalloc_array(num,  in vring_create_virtqueue_packed()
1739 	if (!vq->packed.desc_state)  in vring_create_virtqueue_packed()
1742 	memset(vq->packed.desc_state, 0,  in vring_create_virtqueue_packed()
1748 	vq->packed.desc_extra = vring_alloc_desc_extra(vq, num);  in vring_create_virtqueue_packed()
1749 	if (!vq->packed.desc_extra)  in vring_create_virtqueue_packed()
1754 		vq->packed.event_flags_shadow = VRING_PACKED_EVENT_FLAG_DISABLE;  in vring_create_virtqueue_packed()
1755 		vq->packed.vring.driver->flags =  in vring_create_virtqueue_packed()
1756 			cpu_to_le16(vq->packed.event_flags_shadow);  in vring_create_virtqueue_packed()
1765 	kfree(vq->packed.desc_state);  in vring_create_virtqueue_packed()
2306 					 vq->packed.ring_size_in_bytes,  in vring_del_virtqueue()
2307 					 vq->packed.vring.desc,  in vring_del_virtqueue()
2308 					 vq->packed.ring_dma_addr);  in vring_del_virtqueue()
2311 					 vq->packed.event_size_in_bytes,  in vring_del_virtqueue()
2312 					 vq->packed.vring.driver,  in vring_del_virtqueue()
2313 					 vq->packed.driver_event_dma_addr);  in vring_del_virtqueue()
2316 					 vq->packed.event_size_in_bytes,  in vring_del_virtqueue()
2317 					 vq->packed.vring.device,  in vring_del_virtqueue()
2318 					 vq->packed.device_event_dma_addr);  in vring_del_virtqueue()
2320 			kfree(vq->packed.desc_state);  in vring_del_virtqueue()
2321 			kfree(vq->packed.desc_extra);  in vring_del_virtqueue()
2376 	return vq->packed_ring ? vq->packed.vring.num : vq->split.vring.num;  in virtqueue_get_vring_size()
2414 		return vq->packed.ring_dma_addr;  in virtqueue_get_desc_addr()
2427 		return vq->packed.driver_event_dma_addr;  in virtqueue_get_avail_addr()
2441 		return vq->packed.device_event_dma_addr;  in virtqueue_get_used_addr()