31 #define BAD_RING(_vq, fmt, args...)				\  argument
33 		dev_err(&(_vq)->vq.vdev->dev,			\
34 			"%s:"fmt, (_vq)->vq.name, ##args);	\
38 #define START_USE(_vq)						\  argument
40 		if ((_vq)->in_use)				\
42 			      (_vq)->vq.name, (_vq)->in_use);	\
43 		(_vq)->in_use = __LINE__;			\
45 #define END_USE(_vq) \  argument
46 	do { BUG_ON(!(_vq)->in_use); (_vq)->in_use = 0; } while(0)
48 #define BAD_RING(_vq, fmt, args...)				\  argument
50 		dev_err(&_vq->vq.vdev->dev,			\
51 			"%s:"fmt, (_vq)->vq.name, ##args);	\
52 		(_vq)->broken = true;				\
116 #define to_vvq(_vq) container_of(_vq, struct vring_virtqueue, vq)  argument
237 static struct vring_desc *alloc_indirect(struct virtqueue *_vq,  in alloc_indirect()  argument
255 		desc[i].next = cpu_to_virtio16(_vq->vdev, i + 1);  in alloc_indirect()
259 static inline int virtqueue_add(struct virtqueue *_vq,  in virtqueue_add()  argument
268 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_add()
305 		desc = alloc_indirect(_vq, total_sg, gfp);  in virtqueue_add()
344 			desc[i].flags = cpu_to_virtio16(_vq->vdev, VRING_DESC_F_NEXT);  in virtqueue_add()
345 			desc[i].addr = cpu_to_virtio64(_vq->vdev, addr);  in virtqueue_add()
346 			desc[i].len = cpu_to_virtio32(_vq->vdev, sg->length);  in virtqueue_add()
348 			i = virtio16_to_cpu(_vq->vdev, desc[i].next);  in virtqueue_add()
357 			desc[i].flags = cpu_to_virtio16(_vq->vdev, VRING_DESC_F_NEXT | VRING_DESC_F_WRITE);  in virtqueue_add()
358 			desc[i].addr = cpu_to_virtio64(_vq->vdev, addr);  in virtqueue_add()
359 			desc[i].len = cpu_to_virtio32(_vq->vdev, sg->length);  in virtqueue_add()
361 			i = virtio16_to_cpu(_vq->vdev, desc[i].next);  in virtqueue_add()
365 	desc[prev].flags &= cpu_to_virtio16(_vq->vdev, ~VRING_DESC_F_NEXT);  in virtqueue_add()
375 		vq->vring.desc[head].flags = cpu_to_virtio16(_vq->vdev, VRING_DESC_F_INDIRECT);  in virtqueue_add()
376 		vq->vring.desc[head].addr = cpu_to_virtio64(_vq->vdev, addr);  in virtqueue_add()
378 		vq->vring.desc[head].len = cpu_to_virtio32(_vq->vdev, total_sg * sizeof(struct vring_desc));  in virtqueue_add()
386 		vq->free_head = virtio16_to_cpu(_vq->vdev, vq->vring.desc[head].next);  in virtqueue_add()
400 	vq->vring.avail->ring[avail] = cpu_to_virtio16(_vq->vdev, head);  in virtqueue_add()
406 	vq->vring.avail->idx = cpu_to_virtio16(_vq->vdev, vq->avail_idx_shadow);  in virtqueue_add()
415 		virtqueue_kick(_vq);  in virtqueue_add()
427 		i = virtio16_to_cpu(_vq->vdev, vq->vring.desc[i].next);  in virtqueue_add()
451 int virtqueue_add_sgs(struct virtqueue *_vq,  in virtqueue_add_sgs()  argument
466 	return virtqueue_add(_vq, sgs, total_sg, out_sgs, in_sgs,  in virtqueue_add_sgs()
550 bool virtqueue_kick_prepare(struct virtqueue *_vq)  in virtqueue_kick_prepare()  argument
552 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_kick_prepare()
574 		needs_kick = vring_need_event(virtio16_to_cpu(_vq->vdev, vring_avail_event(&vq->vring)),  in virtqueue_kick_prepare()
577 		needs_kick = !(vq->vring.used->flags & cpu_to_virtio16(_vq->vdev, VRING_USED_F_NO_NOTIFY));  in virtqueue_kick_prepare()
592 bool virtqueue_notify(struct virtqueue *_vq)  in virtqueue_notify()  argument
594 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_notify()
600 	if (!vq->notify(_vq)) {  in virtqueue_notify()
698 void *virtqueue_get_buf_ctx(struct virtqueue *_vq, unsigned int *len,  in virtqueue_get_buf_ctx()  argument
701 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_get_buf_ctx()
723 	i = virtio32_to_cpu(_vq->vdev, vq->vring.used->ring[last_used].id);  in virtqueue_get_buf_ctx()
724 	*len = virtio32_to_cpu(_vq->vdev, vq->vring.used->ring[last_used].len);  in virtqueue_get_buf_ctx()
745 				cpu_to_virtio16(_vq->vdev, vq->last_used_idx));  in virtqueue_get_buf_ctx()
756 void *virtqueue_get_buf(struct virtqueue *_vq, unsigned int *len)  in virtqueue_get_buf()  argument
758 	return virtqueue_get_buf_ctx(_vq, len, NULL);  in virtqueue_get_buf()
770 void virtqueue_disable_cb(struct virtqueue *_vq)  in virtqueue_disable_cb()  argument
772 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_disable_cb()
777 			vq->vring.avail->flags = cpu_to_virtio16(_vq->vdev, vq->avail_flags_shadow);  in virtqueue_disable_cb()
795 unsigned virtqueue_enable_cb_prepare(struct virtqueue *_vq)  in virtqueue_enable_cb_prepare()  argument
797 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_enable_cb_prepare()
810 			vq->vring.avail->flags = cpu_to_virtio16(_vq->vdev, vq->avail_flags_shadow);  in virtqueue_enable_cb_prepare()
812 	vring_used_event(&vq->vring) = cpu_to_virtio16(_vq->vdev, last_used_idx = vq->last_used_idx);  in virtqueue_enable_cb_prepare()
827 bool virtqueue_poll(struct virtqueue *_vq, unsigned last_used_idx)  in virtqueue_poll()  argument
829 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_poll()
832 	return (u16)last_used_idx != virtio16_to_cpu(_vq->vdev, vq->vring.used->idx);  in virtqueue_poll()
847 bool virtqueue_enable_cb(struct virtqueue *_vq)  in virtqueue_enable_cb()  argument
849 	unsigned last_used_idx = virtqueue_enable_cb_prepare(_vq);  in virtqueue_enable_cb()
850 	return !virtqueue_poll(_vq, last_used_idx);  in virtqueue_enable_cb()
867 bool virtqueue_enable_cb_delayed(struct virtqueue *_vq)  in virtqueue_enable_cb_delayed()  argument
869 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_enable_cb_delayed()
882 			vq->vring.avail->flags = cpu_to_virtio16(_vq->vdev, vq->avail_flags_shadow);  in virtqueue_enable_cb_delayed()
889 			cpu_to_virtio16(_vq->vdev, vq->last_used_idx + bufs));  in virtqueue_enable_cb_delayed()
891 	if (unlikely((u16)(virtio16_to_cpu(_vq->vdev, vq->vring.used->idx) - vq->last_used_idx) > bufs)) {  in virtqueue_enable_cb_delayed()
909 void *virtqueue_detach_unused_buf(struct virtqueue *_vq)  in virtqueue_detach_unused_buf()  argument
911 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_detach_unused_buf()
924 		vq->vring.avail->idx = cpu_to_virtio16(_vq->vdev, vq->avail_idx_shadow);  in virtqueue_detach_unused_buf()
936 irqreturn_t vring_interrupt(int irq, void *_vq)  in vring_interrupt()  argument
938 	struct vring_virtqueue *vq = to_vvq(_vq);  in vring_interrupt()
1139 void vring_del_virtqueue(struct virtqueue *_vq)  in vring_del_virtqueue()  argument
1141 	struct vring_virtqueue *vq = to_vvq(_vq);  in vring_del_virtqueue()
1147 	list_del(&_vq->list);  in vring_del_virtqueue()
1182 unsigned int virtqueue_get_vring_size(struct virtqueue *_vq)  in virtqueue_get_vring_size()  argument
1185 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_get_vring_size()
1191 bool virtqueue_is_broken(struct virtqueue *_vq)  in virtqueue_is_broken()  argument
1193 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_is_broken()
1205 	struct virtqueue *_vq;  in virtio_break_device()  local
1207 	list_for_each_entry(_vq, &dev->vqs, list) {  in virtio_break_device()
1208 		struct vring_virtqueue *vq = to_vvq(_vq);  in virtio_break_device()
1214 dma_addr_t virtqueue_get_desc_addr(struct virtqueue *_vq)  in virtqueue_get_desc_addr()  argument
1216 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_get_desc_addr()
1224 dma_addr_t virtqueue_get_avail_addr(struct virtqueue *_vq)  in virtqueue_get_avail_addr()  argument
1226 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_get_avail_addr()
1235 dma_addr_t virtqueue_get_used_addr(struct virtqueue *_vq)  in virtqueue_get_used_addr()  argument
1237 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_get_used_addr()