20 #define BAD_RING(_vq, fmt, args...)				\  argument
22 		dev_err(&(_vq)->vq.vdev->dev,			\
23 			"%s:"fmt, (_vq)->vq.name, ##args);	\
27 #define START_USE(_vq)						\  argument
29 		if ((_vq)->in_use)				\
31 			      (_vq)->vq.name, (_vq)->in_use);	\
32 		(_vq)->in_use = __LINE__;			\
34 #define END_USE(_vq) \  argument
35 	do { BUG_ON(!(_vq)->in_use); (_vq)->in_use = 0; } while(0)
36 #define LAST_ADD_TIME_UPDATE(_vq)				\  argument
41 		if ((_vq)->last_add_time_valid)			\
43 				(_vq)->last_add_time)) > 100);	\
44 		(_vq)->last_add_time = now;			\
45 		(_vq)->last_add_time_valid = true;		\
47 #define LAST_ADD_TIME_CHECK(_vq)				\  argument
49 		if ((_vq)->last_add_time_valid) {		\
51 				      (_vq)->last_add_time)) > 100); \
54 #define LAST_ADD_TIME_INVALID(_vq)				\  argument
55 	((_vq)->last_add_time_valid = false)
57 #define BAD_RING(_vq, fmt, args...)				\  argument
59 		dev_err(&_vq->vq.vdev->dev,			\
60 			"%s:"fmt, (_vq)->vq.name, ##args);	\
61 		(_vq)->broken = true;				\
224 static void vring_free(struct virtqueue *_vq);
230 #define to_vvq(_vq) container_of(_vq, struct vring_virtqueue, vq)  argument
465 static struct vring_desc *alloc_indirect_split(struct virtqueue *_vq,  in alloc_indirect_split()  argument
484 		desc[i].next = cpu_to_virtio16(_vq->vdev, i + 1);  in alloc_indirect_split()
517 static inline int virtqueue_add_split(struct virtqueue *_vq,  in virtqueue_add_split()  argument
526 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_add_split()
550 		desc = alloc_indirect_split(_vq, total_sg, gfp);  in virtqueue_add_split()
593 			i = virtqueue_add_desc_split(_vq, desc, i, addr, sg->length,  in virtqueue_add_split()
608 			i = virtqueue_add_desc_split(_vq, desc, i, addr,  in virtqueue_add_split()
616 	desc[prev].flags &= cpu_to_virtio16(_vq->vdev, ~VRING_DESC_F_NEXT);  in virtqueue_add_split()
629 		virtqueue_add_desc_split(_vq, vq->split.vring.desc,  in virtqueue_add_split()
655 	vq->split.vring.avail->ring[avail] = cpu_to_virtio16(_vq->vdev, head);  in virtqueue_add_split()
661 	vq->split.vring.avail->idx = cpu_to_virtio16(_vq->vdev,  in virtqueue_add_split()
671 		virtqueue_kick(_vq);  in virtqueue_add_split()
688 			i = virtio16_to_cpu(_vq->vdev, desc[i].next);  in virtqueue_add_split()
700 static bool virtqueue_kick_prepare_split(struct virtqueue *_vq)  in virtqueue_kick_prepare_split()  argument
702 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_kick_prepare_split()
719 		needs_kick = vring_need_event(virtio16_to_cpu(_vq->vdev,  in virtqueue_kick_prepare_split()
724 					cpu_to_virtio16(_vq->vdev,  in virtqueue_kick_prepare_split()
787 static void *virtqueue_get_buf_ctx_split(struct virtqueue *_vq,  in virtqueue_get_buf_ctx_split()  argument
791 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_get_buf_ctx_split()
813 	i = virtio32_to_cpu(_vq->vdev,  in virtqueue_get_buf_ctx_split()
815 	*len = virtio32_to_cpu(_vq->vdev,  in virtqueue_get_buf_ctx_split()
837 				cpu_to_virtio16(_vq->vdev, vq->last_used_idx));  in virtqueue_get_buf_ctx_split()
845 static void virtqueue_disable_cb_split(struct virtqueue *_vq)  in virtqueue_disable_cb_split()  argument
847 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_disable_cb_split()
856 				cpu_to_virtio16(_vq->vdev,  in virtqueue_disable_cb_split()
861 static unsigned int virtqueue_enable_cb_prepare_split(struct virtqueue *_vq)  in virtqueue_enable_cb_prepare_split()  argument
863 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_enable_cb_prepare_split()
877 				cpu_to_virtio16(_vq->vdev,  in virtqueue_enable_cb_prepare_split()
880 	vring_used_event(&vq->split.vring) = cpu_to_virtio16(_vq->vdev,  in virtqueue_enable_cb_prepare_split()
886 static bool virtqueue_poll_split(struct virtqueue *_vq, unsigned int last_used_idx)  in virtqueue_poll_split()  argument
888 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_poll_split()
890 	return (u16)last_used_idx != virtio16_to_cpu(_vq->vdev,  in virtqueue_poll_split()
894 static bool virtqueue_enable_cb_delayed_split(struct virtqueue *_vq)  in virtqueue_enable_cb_delayed_split()  argument
896 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_enable_cb_delayed_split()
910 				cpu_to_virtio16(_vq->vdev,  in virtqueue_enable_cb_delayed_split()
918 			cpu_to_virtio16(_vq->vdev, vq->last_used_idx + bufs));  in virtqueue_enable_cb_delayed_split()
920 	if (unlikely((u16)(virtio16_to_cpu(_vq->vdev, vq->split.vring.used->idx)  in virtqueue_enable_cb_delayed_split()
930 static void *virtqueue_detach_unused_buf_split(struct virtqueue *_vq)  in virtqueue_detach_unused_buf_split()  argument
932 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_detach_unused_buf_split()
945 		vq->split.vring.avail->idx = cpu_to_virtio16(_vq->vdev,  in virtqueue_detach_unused_buf_split()
1126 static int virtqueue_resize_split(struct virtqueue *_vq, u32 num)  in virtqueue_resize_split()  argument
1129 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_resize_split()
1130 	struct virtio_device *vdev = _vq->vdev;  in virtqueue_resize_split()
1344 static inline int virtqueue_add_packed(struct virtqueue *_vq,  in virtqueue_add_packed()  argument
1353 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_add_packed()
1495 static bool virtqueue_kick_prepare_packed(struct virtqueue *_vq)  in virtqueue_kick_prepare_packed()  argument
1497 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_kick_prepare_packed()
1615 static void *virtqueue_get_buf_ctx_packed(struct virtqueue *_vq,  in virtqueue_get_buf_ctx_packed()  argument
1619 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_get_buf_ctx_packed()
1684 static void virtqueue_disable_cb_packed(struct virtqueue *_vq)  in virtqueue_disable_cb_packed()  argument
1686 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_disable_cb_packed()
1695 static unsigned int virtqueue_enable_cb_prepare_packed(struct virtqueue *_vq)  in virtqueue_enable_cb_prepare_packed()  argument
1697 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_enable_cb_prepare_packed()
1728 static bool virtqueue_poll_packed(struct virtqueue *_vq, u16 off_wrap)  in virtqueue_poll_packed()  argument
1730 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_poll_packed()
1740 static bool virtqueue_enable_cb_delayed_packed(struct virtqueue *_vq)  in virtqueue_enable_cb_delayed_packed()  argument
1742 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_enable_cb_delayed_packed()
1801 static void *virtqueue_detach_unused_buf_packed(struct virtqueue *_vq)  in virtqueue_detach_unused_buf_packed()  argument
1803 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_detach_unused_buf_packed()
2048 static int virtqueue_resize_packed(struct virtqueue *_vq, u32 num)  in virtqueue_resize_packed()  argument
2051 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_resize_packed()
2052 	struct virtio_device *vdev = _vq->vdev;  in virtqueue_resize_packed()
2083 static inline int virtqueue_add(struct virtqueue *_vq,  in virtqueue_add()  argument
2092 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_add()
2094 	return vq->packed_ring ? virtqueue_add_packed(_vq, sgs, total_sg,  in virtqueue_add()
2096 				 virtqueue_add_split(_vq, sgs, total_sg,  in virtqueue_add()
2114 int virtqueue_add_sgs(struct virtqueue *_vq,  in virtqueue_add_sgs()  argument
2130 	return virtqueue_add(_vq, sgs, total_sg, out_sgs, in_sgs,  in virtqueue_add_sgs()
2214 bool virtqueue_kick_prepare(struct virtqueue *_vq)  in virtqueue_kick_prepare()  argument
2216 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_kick_prepare()
2218 	return vq->packed_ring ? virtqueue_kick_prepare_packed(_vq) :  in virtqueue_kick_prepare()
2219 				 virtqueue_kick_prepare_split(_vq);  in virtqueue_kick_prepare()
2231 bool virtqueue_notify(struct virtqueue *_vq)  in virtqueue_notify()  argument
2233 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_notify()
2239 	if (!vq->notify(_vq)) {  in virtqueue_notify()
2284 void *virtqueue_get_buf_ctx(struct virtqueue *_vq, unsigned int *len,  in virtqueue_get_buf_ctx()  argument
2287 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_get_buf_ctx()
2289 	return vq->packed_ring ? virtqueue_get_buf_ctx_packed(_vq, len, ctx) :  in virtqueue_get_buf_ctx()
2290 				 virtqueue_get_buf_ctx_split(_vq, len, ctx);  in virtqueue_get_buf_ctx()
2294 void *virtqueue_get_buf(struct virtqueue *_vq, unsigned int *len)  in virtqueue_get_buf()  argument
2296 	return virtqueue_get_buf_ctx(_vq, len, NULL);  in virtqueue_get_buf()
2308 void virtqueue_disable_cb(struct virtqueue *_vq)  in virtqueue_disable_cb()  argument
2310 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_disable_cb()
2319 		virtqueue_disable_cb_packed(_vq);  in virtqueue_disable_cb()
2321 		virtqueue_disable_cb_split(_vq);  in virtqueue_disable_cb()
2337 unsigned int virtqueue_enable_cb_prepare(struct virtqueue *_vq)  in virtqueue_enable_cb_prepare()  argument
2339 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_enable_cb_prepare()
2344 	return vq->packed_ring ? virtqueue_enable_cb_prepare_packed(_vq) :  in virtqueue_enable_cb_prepare()
2345 				 virtqueue_enable_cb_prepare_split(_vq);  in virtqueue_enable_cb_prepare()
2358 bool virtqueue_poll(struct virtqueue *_vq, unsigned int last_used_idx)  in virtqueue_poll()  argument
2360 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_poll()
2366 	return vq->packed_ring ? virtqueue_poll_packed(_vq, last_used_idx) :  in virtqueue_poll()
2367 				 virtqueue_poll_split(_vq, last_used_idx);  in virtqueue_poll()
2382 bool virtqueue_enable_cb(struct virtqueue *_vq)  in virtqueue_enable_cb()  argument
2384 	unsigned int last_used_idx = virtqueue_enable_cb_prepare(_vq);  in virtqueue_enable_cb()
2386 	return !virtqueue_poll(_vq, last_used_idx);  in virtqueue_enable_cb()
2403 bool virtqueue_enable_cb_delayed(struct virtqueue *_vq)  in virtqueue_enable_cb_delayed()  argument
2405 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_enable_cb_delayed()
2410 	return vq->packed_ring ? virtqueue_enable_cb_delayed_packed(_vq) :  in virtqueue_enable_cb_delayed()
2411 				 virtqueue_enable_cb_delayed_split(_vq);  in virtqueue_enable_cb_delayed()
2423 void *virtqueue_detach_unused_buf(struct virtqueue *_vq)  in virtqueue_detach_unused_buf()  argument
2425 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_detach_unused_buf()
2427 	return vq->packed_ring ? virtqueue_detach_unused_buf_packed(_vq) :  in virtqueue_detach_unused_buf()
2428 				 virtqueue_detach_unused_buf_split(_vq);  in virtqueue_detach_unused_buf()
2445 irqreturn_t vring_interrupt(int irq, void *_vq)  in vring_interrupt()  argument
2447 	struct vring_virtqueue *vq = to_vvq(_vq);  in vring_interrupt()
2584 int virtqueue_resize(struct virtqueue *_vq, u32 num,  in virtqueue_resize()  argument
2587 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_resize()
2610 	err = vdev->config->disable_vq_and_reset(_vq);  in virtqueue_resize()
2614 	while ((buf = virtqueue_detach_unused_buf(_vq)) != NULL)  in virtqueue_resize()
2615 		recycle(_vq, buf);  in virtqueue_resize()
2618 		err = virtqueue_resize_packed(_vq, num);  in virtqueue_resize()
2620 		err = virtqueue_resize_split(_vq, num);  in virtqueue_resize()
2622 	if (vdev->config->enable_vq_after_reset(_vq))  in virtqueue_resize()
2652 static void vring_free(struct virtqueue *_vq)  in vring_free()  argument
2654 	struct vring_virtqueue *vq = to_vvq(_vq);  in vring_free()
2688 void vring_del_virtqueue(struct virtqueue *_vq)  in vring_del_virtqueue()  argument
2690 	struct vring_virtqueue *vq = to_vvq(_vq);  in vring_del_virtqueue()
2693 	list_del(&_vq->list);  in vring_del_virtqueue()
2696 	vring_free(_vq);  in vring_del_virtqueue()
2736 unsigned int virtqueue_get_vring_size(struct virtqueue *_vq)  in virtqueue_get_vring_size()  argument
2739 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_get_vring_size()
2748 void __virtqueue_break(struct virtqueue *_vq)  in __virtqueue_break()  argument
2750 	struct vring_virtqueue *vq = to_vvq(_vq);  in __virtqueue_break()
2760 void __virtqueue_unbreak(struct virtqueue *_vq)  in __virtqueue_unbreak()  argument
2762 	struct vring_virtqueue *vq = to_vvq(_vq);  in __virtqueue_unbreak()
2769 bool virtqueue_is_broken(struct virtqueue *_vq)  in virtqueue_is_broken()  argument
2771 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_is_broken()
2783 	struct virtqueue *_vq;  in virtio_break_device()  local
2786 	list_for_each_entry(_vq, &dev->vqs, list) {  in virtio_break_device()
2787 		struct vring_virtqueue *vq = to_vvq(_vq);  in virtio_break_device()
2805 	struct virtqueue *_vq;  in __virtio_unbreak_device()  local
2808 	list_for_each_entry(_vq, &dev->vqs, list) {  in __virtio_unbreak_device()
2809 		struct vring_virtqueue *vq = to_vvq(_vq);  in __virtio_unbreak_device()
2818 dma_addr_t virtqueue_get_desc_addr(struct virtqueue *_vq)  in virtqueue_get_desc_addr()  argument
2820 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_get_desc_addr()
2831 dma_addr_t virtqueue_get_avail_addr(struct virtqueue *_vq)  in virtqueue_get_avail_addr()  argument
2833 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_get_avail_addr()
2845 dma_addr_t virtqueue_get_used_addr(struct virtqueue *_vq)  in virtqueue_get_used_addr()  argument
2847 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_get_used_addr()