18 #define BAD_RING(_vq, fmt, args...)				\  argument
20 		dev_err(&(_vq)->vq.vdev->dev,			\
21 			"%s:"fmt, (_vq)->vq.name, ##args);	\
25 #define START_USE(_vq)						\  argument
27 		if ((_vq)->in_use)				\
29 			      (_vq)->vq.name, (_vq)->in_use);	\
30 		(_vq)->in_use = __LINE__;			\
32 #define END_USE(_vq) \  argument
33 	do { BUG_ON(!(_vq)->in_use); (_vq)->in_use = 0; } while(0)
34 #define LAST_ADD_TIME_UPDATE(_vq)				\  argument
39 		if ((_vq)->last_add_time_valid)			\
41 				(_vq)->last_add_time)) > 100);	\
42 		(_vq)->last_add_time = now;			\
43 		(_vq)->last_add_time_valid = true;		\
45 #define LAST_ADD_TIME_CHECK(_vq)				\  argument
47 		if ((_vq)->last_add_time_valid) {		\
49 				      (_vq)->last_add_time)) > 100); \
52 #define LAST_ADD_TIME_INVALID(_vq)				\  argument
53 	((_vq)->last_add_time_valid = false)
55 #define BAD_RING(_vq, fmt, args...)				\  argument
57 		dev_err(&_vq->vq.vdev->dev,			\
58 			"%s:"fmt, (_vq)->vq.name, ##args);	\
59 		(_vq)->broken = true;				\
201 #define to_vvq(_vq) container_of(_vq, struct vring_virtqueue, vq)  argument
203 static inline bool virtqueue_use_indirect(struct virtqueue *_vq,  in virtqueue_use_indirect()  argument
206 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_use_indirect()
392 static struct vring_desc *alloc_indirect_split(struct virtqueue *_vq,  in alloc_indirect_split()  argument
411 		desc[i].next = cpu_to_virtio16(_vq->vdev, i + 1);  in alloc_indirect_split()
415 static inline int virtqueue_add_split(struct virtqueue *_vq,  in virtqueue_add_split()  argument
424 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_add_split()
447 	if (virtqueue_use_indirect(_vq, total_sg))  in virtqueue_add_split()
448 		desc = alloc_indirect_split(_vq, total_sg, gfp);  in virtqueue_add_split()
487 			desc[i].flags = cpu_to_virtio16(_vq->vdev, VRING_DESC_F_NEXT);  in virtqueue_add_split()
488 			desc[i].addr = cpu_to_virtio64(_vq->vdev, addr);  in virtqueue_add_split()
489 			desc[i].len = cpu_to_virtio32(_vq->vdev, sg->length);  in virtqueue_add_split()
491 			i = virtio16_to_cpu(_vq->vdev, desc[i].next);  in virtqueue_add_split()
500 			desc[i].flags = cpu_to_virtio16(_vq->vdev, VRING_DESC_F_NEXT | VRING_DESC_F_WRITE);  in virtqueue_add_split()
501 			desc[i].addr = cpu_to_virtio64(_vq->vdev, addr);  in virtqueue_add_split()
502 			desc[i].len = cpu_to_virtio32(_vq->vdev, sg->length);  in virtqueue_add_split()
504 			i = virtio16_to_cpu(_vq->vdev, desc[i].next);  in virtqueue_add_split()
508 	desc[prev].flags &= cpu_to_virtio16(_vq->vdev, ~VRING_DESC_F_NEXT);  in virtqueue_add_split()
518 		vq->split.vring.desc[head].flags = cpu_to_virtio16(_vq->vdev,  in virtqueue_add_split()
520 		vq->split.vring.desc[head].addr = cpu_to_virtio64(_vq->vdev,  in virtqueue_add_split()
523 		vq->split.vring.desc[head].len = cpu_to_virtio32(_vq->vdev,  in virtqueue_add_split()
532 		vq->free_head = virtio16_to_cpu(_vq->vdev,  in virtqueue_add_split()
547 	vq->split.vring.avail->ring[avail] = cpu_to_virtio16(_vq->vdev, head);  in virtqueue_add_split()
553 	vq->split.vring.avail->idx = cpu_to_virtio16(_vq->vdev,  in virtqueue_add_split()
563 		virtqueue_kick(_vq);  in virtqueue_add_split()
579 		i = virtio16_to_cpu(_vq->vdev, desc[i].next);  in virtqueue_add_split()
589 static bool virtqueue_kick_prepare_split(struct virtqueue *_vq)  in virtqueue_kick_prepare_split()  argument
591 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_kick_prepare_split()
608 		needs_kick = vring_need_event(virtio16_to_cpu(_vq->vdev,  in virtqueue_kick_prepare_split()
613 					cpu_to_virtio16(_vq->vdev,  in virtqueue_kick_prepare_split()
678 static void *virtqueue_get_buf_ctx_split(struct virtqueue *_vq,  in virtqueue_get_buf_ctx_split()  argument
682 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_get_buf_ctx_split()
704 	i = virtio32_to_cpu(_vq->vdev,  in virtqueue_get_buf_ctx_split()
706 	*len = virtio32_to_cpu(_vq->vdev,  in virtqueue_get_buf_ctx_split()
728 				cpu_to_virtio16(_vq->vdev, vq->last_used_idx));  in virtqueue_get_buf_ctx_split()
736 static void virtqueue_disable_cb_split(struct virtqueue *_vq)  in virtqueue_disable_cb_split()  argument
738 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_disable_cb_split()
744 				cpu_to_virtio16(_vq->vdev,  in virtqueue_disable_cb_split()
749 static unsigned virtqueue_enable_cb_prepare_split(struct virtqueue *_vq)  in virtqueue_enable_cb_prepare_split()  argument
751 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_enable_cb_prepare_split()
765 				cpu_to_virtio16(_vq->vdev,  in virtqueue_enable_cb_prepare_split()
768 	vring_used_event(&vq->split.vring) = cpu_to_virtio16(_vq->vdev,  in virtqueue_enable_cb_prepare_split()
774 static bool virtqueue_poll_split(struct virtqueue *_vq, unsigned last_used_idx)  in virtqueue_poll_split()  argument
776 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_poll_split()
778 	return (u16)last_used_idx != virtio16_to_cpu(_vq->vdev,  in virtqueue_poll_split()
782 static bool virtqueue_enable_cb_delayed_split(struct virtqueue *_vq)  in virtqueue_enable_cb_delayed_split()  argument
784 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_enable_cb_delayed_split()
798 				cpu_to_virtio16(_vq->vdev,  in virtqueue_enable_cb_delayed_split()
806 			cpu_to_virtio16(_vq->vdev, vq->last_used_idx + bufs));  in virtqueue_enable_cb_delayed_split()
808 	if (unlikely((u16)(virtio16_to_cpu(_vq->vdev, vq->split.vring.used->idx)  in virtqueue_enable_cb_delayed_split()
818 static void *virtqueue_detach_unused_buf_split(struct virtqueue *_vq)  in virtqueue_detach_unused_buf_split()  argument
820 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_detach_unused_buf_split()
833 		vq->split.vring.avail->idx = cpu_to_virtio16(_vq->vdev,  in virtqueue_detach_unused_buf_split()
1091 static inline int virtqueue_add_packed(struct virtqueue *_vq,  in virtqueue_add_packed()  argument
1100 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_add_packed()
1121 	if (virtqueue_use_indirect(_vq, total_sg))  in virtqueue_add_packed()
1232 static bool virtqueue_kick_prepare_packed(struct virtqueue *_vq)  in virtqueue_kick_prepare_packed()  argument
1234 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_kick_prepare_packed()
1346 static void *virtqueue_get_buf_ctx_packed(struct virtqueue *_vq,  in virtqueue_get_buf_ctx_packed()  argument
1350 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_get_buf_ctx_packed()
1411 static void virtqueue_disable_cb_packed(struct virtqueue *_vq)  in virtqueue_disable_cb_packed()  argument
1413 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_disable_cb_packed()
1422 static unsigned virtqueue_enable_cb_prepare_packed(struct virtqueue *_vq)  in virtqueue_enable_cb_prepare_packed()  argument
1424 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_enable_cb_prepare_packed()
1458 static bool virtqueue_poll_packed(struct virtqueue *_vq, u16 off_wrap)  in virtqueue_poll_packed()  argument
1460 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_poll_packed()
1470 static bool virtqueue_enable_cb_delayed_packed(struct virtqueue *_vq)  in virtqueue_enable_cb_delayed_packed()  argument
1472 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_enable_cb_delayed_packed()
1529 static void *virtqueue_detach_unused_buf_packed(struct virtqueue *_vq)  in virtqueue_detach_unused_buf_packed()  argument
1531 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_detach_unused_buf_packed()
1693 static inline int virtqueue_add(struct virtqueue *_vq,  in virtqueue_add()  argument
1702 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_add()
1704 	return vq->packed_ring ? virtqueue_add_packed(_vq, sgs, total_sg,  in virtqueue_add()
1706 				 virtqueue_add_split(_vq, sgs, total_sg,  in virtqueue_add()
1724 int virtqueue_add_sgs(struct virtqueue *_vq,  in virtqueue_add_sgs()  argument
1740 	return virtqueue_add(_vq, sgs, total_sg, out_sgs, in_sgs,  in virtqueue_add_sgs()
1824 bool virtqueue_kick_prepare(struct virtqueue *_vq)  in virtqueue_kick_prepare()  argument
1826 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_kick_prepare()
1828 	return vq->packed_ring ? virtqueue_kick_prepare_packed(_vq) :  in virtqueue_kick_prepare()
1829 				 virtqueue_kick_prepare_split(_vq);  in virtqueue_kick_prepare()
1841 bool virtqueue_notify(struct virtqueue *_vq)  in virtqueue_notify()  argument
1843 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_notify()
1849 	if (!vq->notify(_vq)) {  in virtqueue_notify()
1894 void *virtqueue_get_buf_ctx(struct virtqueue *_vq, unsigned int *len,  in virtqueue_get_buf_ctx()  argument
1897 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_get_buf_ctx()
1899 	return vq->packed_ring ? virtqueue_get_buf_ctx_packed(_vq, len, ctx) :  in virtqueue_get_buf_ctx()
1900 				 virtqueue_get_buf_ctx_split(_vq, len, ctx);  in virtqueue_get_buf_ctx()
1904 void *virtqueue_get_buf(struct virtqueue *_vq, unsigned int *len)  in virtqueue_get_buf()  argument
1906 	return virtqueue_get_buf_ctx(_vq, len, NULL);  in virtqueue_get_buf()
1918 void virtqueue_disable_cb(struct virtqueue *_vq)  in virtqueue_disable_cb()  argument
1920 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_disable_cb()
1923 		virtqueue_disable_cb_packed(_vq);  in virtqueue_disable_cb()
1925 		virtqueue_disable_cb_split(_vq);  in virtqueue_disable_cb()
1941 unsigned virtqueue_enable_cb_prepare(struct virtqueue *_vq)  in virtqueue_enable_cb_prepare()  argument
1943 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_enable_cb_prepare()
1945 	return vq->packed_ring ? virtqueue_enable_cb_prepare_packed(_vq) :  in virtqueue_enable_cb_prepare()
1946 				 virtqueue_enable_cb_prepare_split(_vq);  in virtqueue_enable_cb_prepare()
1959 bool virtqueue_poll(struct virtqueue *_vq, unsigned last_used_idx)  in virtqueue_poll()  argument
1961 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_poll()
1964 	return vq->packed_ring ? virtqueue_poll_packed(_vq, last_used_idx) :  in virtqueue_poll()
1965 				 virtqueue_poll_split(_vq, last_used_idx);  in virtqueue_poll()
1980 bool virtqueue_enable_cb(struct virtqueue *_vq)  in virtqueue_enable_cb()  argument
1982 	unsigned last_used_idx = virtqueue_enable_cb_prepare(_vq);  in virtqueue_enable_cb()
1984 	return !virtqueue_poll(_vq, last_used_idx);  in virtqueue_enable_cb()
2001 bool virtqueue_enable_cb_delayed(struct virtqueue *_vq)  in virtqueue_enable_cb_delayed()  argument
2003 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_enable_cb_delayed()
2005 	return vq->packed_ring ? virtqueue_enable_cb_delayed_packed(_vq) :  in virtqueue_enable_cb_delayed()
2006 				 virtqueue_enable_cb_delayed_split(_vq);  in virtqueue_enable_cb_delayed()
2018 void *virtqueue_detach_unused_buf(struct virtqueue *_vq)  in virtqueue_detach_unused_buf()  argument
2020 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_detach_unused_buf()
2022 	return vq->packed_ring ? virtqueue_detach_unused_buf_packed(_vq) :  in virtqueue_detach_unused_buf()
2023 				 virtqueue_detach_unused_buf_split(_vq);  in virtqueue_detach_unused_buf()
2032 irqreturn_t vring_interrupt(int irq, void *_vq)  in vring_interrupt()  argument
2034 	struct vring_virtqueue *vq = to_vvq(_vq);  in vring_interrupt()
2178 void vring_del_virtqueue(struct virtqueue *_vq)  in vring_del_virtqueue()  argument
2180 	struct vring_virtqueue *vq = to_vvq(_vq);  in vring_del_virtqueue()
2210 	list_del(&_vq->list);  in vring_del_virtqueue()
2249 unsigned int virtqueue_get_vring_size(struct virtqueue *_vq)  in virtqueue_get_vring_size()  argument
2252 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_get_vring_size()
2258 bool virtqueue_is_broken(struct virtqueue *_vq)  in virtqueue_is_broken()  argument
2260 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_is_broken()
2272 	struct virtqueue *_vq;  in virtio_break_device()  local
2274 	list_for_each_entry(_vq, &dev->vqs, list) {  in virtio_break_device()
2275 		struct vring_virtqueue *vq = to_vvq(_vq);  in virtio_break_device()
2281 dma_addr_t virtqueue_get_desc_addr(struct virtqueue *_vq)  in virtqueue_get_desc_addr()  argument
2283 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_get_desc_addr()
2294 dma_addr_t virtqueue_get_avail_addr(struct virtqueue *_vq)  in virtqueue_get_avail_addr()  argument
2296 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_get_avail_addr()
2308 dma_addr_t virtqueue_get_used_addr(struct virtqueue *_vq)  in virtqueue_get_used_addr()  argument
2310 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_get_used_addr()