33 		dev_err(&(_vq)->vq.vdev->dev,			\
34 			"%s:"fmt, (_vq)->vq.name, ##args);	\
42 			      (_vq)->vq.name, (_vq)->in_use);	\
50 		dev_err(&_vq->vq.vdev->dev,			\
51 			"%s:"fmt, (_vq)->vq.name, ##args);	\
54 #define START_USE(vq)  argument
55 #define END_USE(vq)  argument
64 	struct virtqueue vq;  member
96 	bool (*notify)(struct virtqueue *vq);
116 #define to_vvq(_vq) container_of(_vq, struct vring_virtqueue, vq)
169 static inline struct device *vring_dma_dev(const struct vring_virtqueue *vq)  in vring_dma_dev()  argument
171 	return vq->vq.vdev->dev.parent;  in vring_dma_dev()
175 static dma_addr_t vring_map_one_sg(const struct vring_virtqueue *vq,  in vring_map_one_sg()  argument
179 	if (!vring_use_dma_api(vq->vq.vdev))  in vring_map_one_sg()
187 	return dma_map_page(vring_dma_dev(vq),  in vring_map_one_sg()
192 static dma_addr_t vring_map_single(const struct vring_virtqueue *vq,  in vring_map_single()  argument
196 	if (!vring_use_dma_api(vq->vq.vdev))  in vring_map_single()
199 	return dma_map_single(vring_dma_dev(vq),  in vring_map_single()
203 static void vring_unmap_one(const struct vring_virtqueue *vq,  in vring_unmap_one()  argument
208 	if (!vring_use_dma_api(vq->vq.vdev))  in vring_unmap_one()
211 	flags = virtio16_to_cpu(vq->vq.vdev, desc->flags);  in vring_unmap_one()
214 		dma_unmap_single(vring_dma_dev(vq),  in vring_unmap_one()
215 				 virtio64_to_cpu(vq->vq.vdev, desc->addr),  in vring_unmap_one()
216 				 virtio32_to_cpu(vq->vq.vdev, desc->len),  in vring_unmap_one()
220 		dma_unmap_page(vring_dma_dev(vq),  in vring_unmap_one()
221 			       virtio64_to_cpu(vq->vq.vdev, desc->addr),  in vring_unmap_one()
222 			       virtio32_to_cpu(vq->vq.vdev, desc->len),  in vring_unmap_one()
228 static int vring_mapping_error(const struct vring_virtqueue *vq,  in vring_mapping_error()  argument
231 	if (!vring_use_dma_api(vq->vq.vdev))  in vring_mapping_error()
234 	return dma_mapping_error(vring_dma_dev(vq), addr);  in vring_mapping_error()
268 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_add()  local
275 	START_USE(vq);  in virtqueue_add()
278 	BUG_ON(ctx && vq->indirect);  in virtqueue_add()
280 	if (unlikely(vq->broken)) {  in virtqueue_add()
281 		END_USE(vq);  in virtqueue_add()
290 		if (vq->last_add_time_valid)  in virtqueue_add()
291 			WARN_ON(ktime_to_ms(ktime_sub(now, vq->last_add_time))  in virtqueue_add()
293 		vq->last_add_time = now;  in virtqueue_add()
294 		vq->last_add_time_valid = true;  in virtqueue_add()
300 	head = vq->free_head;  in virtqueue_add()
304 	if (vq->indirect && total_sg > 1 && vq->vq.num_free)  in virtqueue_add()
308 		WARN_ON_ONCE(total_sg > vq->vring.num && !vq->indirect);  in virtqueue_add()
319 		desc = vq->vring.desc;  in virtqueue_add()
324 	if (vq->vq.num_free < descs_used) {  in virtqueue_add()
326 			 descs_used, vq->vq.num_free);  in virtqueue_add()
331 			vq->notify(&vq->vq);  in virtqueue_add()
334 		END_USE(vq);  in virtqueue_add()
340 			dma_addr_t addr = vring_map_one_sg(vq, sg, DMA_TO_DEVICE);  in virtqueue_add()
341 			if (vring_mapping_error(vq, addr))  in virtqueue_add()
353 			dma_addr_t addr = vring_map_one_sg(vq, sg, DMA_FROM_DEVICE);  in virtqueue_add()
354 			if (vring_mapping_error(vq, addr))  in virtqueue_add()
370 			vq, desc, total_sg * sizeof(struct vring_desc),  in virtqueue_add()
372 		if (vring_mapping_error(vq, addr))  in virtqueue_add()
375 		vq->vring.desc[head].flags = cpu_to_virtio16(_vq->vdev, VRING_DESC_F_INDIRECT);  in virtqueue_add()
376 		vq->vring.desc[head].addr = cpu_to_virtio64(_vq->vdev, addr);  in virtqueue_add()
378 		vq->vring.desc[head].len = cpu_to_virtio32(_vq->vdev, total_sg * sizeof(struct vring_desc));  in virtqueue_add()
382 	vq->vq.num_free -= descs_used;  in virtqueue_add()
386 		vq->free_head = virtio16_to_cpu(_vq->vdev, vq->vring.desc[head].next);  in virtqueue_add()
388 		vq->free_head = i;  in virtqueue_add()
391 	vq->desc_state[head].data = data;  in virtqueue_add()
393 		vq->desc_state[head].indir_desc = desc;  in virtqueue_add()
395 		vq->desc_state[head].indir_desc = ctx;  in virtqueue_add()
399 	avail = vq->avail_idx_shadow & (vq->vring.num - 1);  in virtqueue_add()
400 	vq->vring.avail->ring[avail] = cpu_to_virtio16(_vq->vdev, head);  in virtqueue_add()
404 	virtio_wmb(vq->weak_barriers);  in virtqueue_add()
405 	vq->avail_idx_shadow++;  in virtqueue_add()
406 	vq->vring.avail->idx = cpu_to_virtio16(_vq->vdev, vq->avail_idx_shadow);  in virtqueue_add()
407 	vq->num_added++;  in virtqueue_add()
409 	pr_debug("Added buffer head %i to %p\n", head, vq);  in virtqueue_add()
410 	END_USE(vq);  in virtqueue_add()
414 	if (unlikely(vq->num_added == (1 << 16) - 1))  in virtqueue_add()
426 		vring_unmap_one(vq, &desc[i]);  in virtqueue_add()
427 		i = virtio16_to_cpu(_vq->vdev, vq->vring.desc[i].next);  in virtqueue_add()
433 	END_USE(vq);  in virtqueue_add()
484 int virtqueue_add_outbuf(struct virtqueue *vq,  in virtqueue_add_outbuf()  argument
489 	return virtqueue_add(vq, &sg, num, 1, 0, data, NULL, gfp);  in virtqueue_add_outbuf()
506 int virtqueue_add_inbuf(struct virtqueue *vq,  in virtqueue_add_inbuf()  argument
511 	return virtqueue_add(vq, &sg, num, 0, 1, data, NULL, gfp);  in virtqueue_add_inbuf()
529 int virtqueue_add_inbuf_ctx(struct virtqueue *vq,  in virtqueue_add_inbuf_ctx()  argument
535 	return virtqueue_add(vq, &sg, num, 0, 1, data, ctx, gfp);  in virtqueue_add_inbuf_ctx()
552 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_kick_prepare()  local
556 	START_USE(vq);  in virtqueue_kick_prepare()
559 	virtio_mb(vq->weak_barriers);  in virtqueue_kick_prepare()
561 	old = vq->avail_idx_shadow - vq->num_added;  in virtqueue_kick_prepare()
562 	new = vq->avail_idx_shadow;  in virtqueue_kick_prepare()
563 	vq->num_added = 0;  in virtqueue_kick_prepare()
566 	if (vq->last_add_time_valid) {  in virtqueue_kick_prepare()
568 					      vq->last_add_time)) > 100);  in virtqueue_kick_prepare()
570 	vq->last_add_time_valid = false;  in virtqueue_kick_prepare()
573 	if (vq->event) {  in virtqueue_kick_prepare()
574 		needs_kick = vring_need_event(virtio16_to_cpu(_vq->vdev, vring_avail_event(&vq->vring)),  in virtqueue_kick_prepare()
577 		needs_kick = !(vq->vring.used->flags & cpu_to_virtio16(_vq->vdev, VRING_USED_F_NO_NOTIFY));  in virtqueue_kick_prepare()
579 	END_USE(vq);  in virtqueue_kick_prepare()
594 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_notify()  local
596 	if (unlikely(vq->broken))  in virtqueue_notify()
600 	if (!vq->notify(_vq)) {  in virtqueue_notify()
601 		vq->broken = true;  in virtqueue_notify()
620 bool virtqueue_kick(struct virtqueue *vq)  in virtqueue_kick()  argument
622 	if (virtqueue_kick_prepare(vq))  in virtqueue_kick()
623 		return virtqueue_notify(vq);  in virtqueue_kick()
628 static void detach_buf(struct vring_virtqueue *vq, unsigned int head,  in detach_buf()  argument
632 	__virtio16 nextflag = cpu_to_virtio16(vq->vq.vdev, VRING_DESC_F_NEXT);  in detach_buf()
635 	vq->desc_state[head].data = NULL;  in detach_buf()
640 	while (vq->vring.desc[i].flags & nextflag) {  in detach_buf()
641 		vring_unmap_one(vq, &vq->vring.desc[i]);  in detach_buf()
642 		i = virtio16_to_cpu(vq->vq.vdev, vq->vring.desc[i].next);  in detach_buf()
643 		vq->vq.num_free++;  in detach_buf()
646 	vring_unmap_one(vq, &vq->vring.desc[i]);  in detach_buf()
647 	vq->vring.desc[i].next = cpu_to_virtio16(vq->vq.vdev, vq->free_head);  in detach_buf()
648 	vq->free_head = head;  in detach_buf()
651 	vq->vq.num_free++;  in detach_buf()
653 	if (vq->indirect) {  in detach_buf()
654 		struct vring_desc *indir_desc = vq->desc_state[head].indir_desc;  in detach_buf()
661 		len = virtio32_to_cpu(vq->vq.vdev, vq->vring.desc[head].len);  in detach_buf()
663 		BUG_ON(!(vq->vring.desc[head].flags &  in detach_buf()
664 			 cpu_to_virtio16(vq->vq.vdev, VRING_DESC_F_INDIRECT)));  in detach_buf()
668 			vring_unmap_one(vq, &indir_desc[j]);  in detach_buf()
671 		vq->desc_state[head].indir_desc = NULL;  in detach_buf()
673 		*ctx = vq->desc_state[head].indir_desc;  in detach_buf()
677 static inline bool more_used(const struct vring_virtqueue *vq)  in more_used()  argument
679 	return vq->last_used_idx != virtio16_to_cpu(vq->vq.vdev, vq->vring.used->idx);  in more_used()
701 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_get_buf_ctx()  local
706 	START_USE(vq);  in virtqueue_get_buf_ctx()
708 	if (unlikely(vq->broken)) {  in virtqueue_get_buf_ctx()
709 		END_USE(vq);  in virtqueue_get_buf_ctx()
713 	if (!more_used(vq)) {  in virtqueue_get_buf_ctx()
715 		END_USE(vq);  in virtqueue_get_buf_ctx()
720 	virtio_rmb(vq->weak_barriers);  in virtqueue_get_buf_ctx()
722 	last_used = (vq->last_used_idx & (vq->vring.num - 1));  in virtqueue_get_buf_ctx()
723 	i = virtio32_to_cpu(_vq->vdev, vq->vring.used->ring[last_used].id);  in virtqueue_get_buf_ctx()
724 	*len = virtio32_to_cpu(_vq->vdev, vq->vring.used->ring[last_used].len);  in virtqueue_get_buf_ctx()
726 	if (unlikely(i >= vq->vring.num)) {  in virtqueue_get_buf_ctx()
727 		BAD_RING(vq, "id %u out of range\n", i);  in virtqueue_get_buf_ctx()
730 	if (unlikely(!vq->desc_state[i].data)) {  in virtqueue_get_buf_ctx()
731 		BAD_RING(vq, "id %u is not a head!\n", i);  in virtqueue_get_buf_ctx()
736 	ret = vq->desc_state[i].data;  in virtqueue_get_buf_ctx()
737 	detach_buf(vq, i, ctx);  in virtqueue_get_buf_ctx()
738 	vq->last_used_idx++;  in virtqueue_get_buf_ctx()
742 	if (!(vq->avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT))  in virtqueue_get_buf_ctx()
743 		virtio_store_mb(vq->weak_barriers,  in virtqueue_get_buf_ctx()
744 				&vring_used_event(&vq->vring),  in virtqueue_get_buf_ctx()
745 				cpu_to_virtio16(_vq->vdev, vq->last_used_idx));  in virtqueue_get_buf_ctx()
748 	vq->last_add_time_valid = false;  in virtqueue_get_buf_ctx()
751 	END_USE(vq);  in virtqueue_get_buf_ctx()
772 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_disable_cb()  local
774 	if (!(vq->avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT)) {  in virtqueue_disable_cb()
775 		vq->avail_flags_shadow |= VRING_AVAIL_F_NO_INTERRUPT;  in virtqueue_disable_cb()
776 		if (!vq->event)  in virtqueue_disable_cb()
777 			vq->vring.avail->flags = cpu_to_virtio16(_vq->vdev, vq->avail_flags_shadow);  in virtqueue_disable_cb()
797 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_enable_cb_prepare()  local
800 	START_USE(vq);  in virtqueue_enable_cb_prepare()
807 	if (vq->avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT) {  in virtqueue_enable_cb_prepare()
808 		vq->avail_flags_shadow &= ~VRING_AVAIL_F_NO_INTERRUPT;  in virtqueue_enable_cb_prepare()
809 		if (!vq->event)  in virtqueue_enable_cb_prepare()
810 			vq->vring.avail->flags = cpu_to_virtio16(_vq->vdev, vq->avail_flags_shadow);  in virtqueue_enable_cb_prepare()
812 	vring_used_event(&vq->vring) = cpu_to_virtio16(_vq->vdev, last_used_idx = vq->last_used_idx);  in virtqueue_enable_cb_prepare()
813 	END_USE(vq);  in virtqueue_enable_cb_prepare()
829 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_poll()  local
831 	virtio_mb(vq->weak_barriers);  in virtqueue_poll()
832 	return (u16)last_used_idx != virtio16_to_cpu(_vq->vdev, vq->vring.used->idx);  in virtqueue_poll()
869 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_enable_cb_delayed()  local
872 	START_USE(vq);  in virtqueue_enable_cb_delayed()
879 	if (vq->avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT) {  in virtqueue_enable_cb_delayed()
880 		vq->avail_flags_shadow &= ~VRING_AVAIL_F_NO_INTERRUPT;  in virtqueue_enable_cb_delayed()
881 		if (!vq->event)  in virtqueue_enable_cb_delayed()
882 			vq->vring.avail->flags = cpu_to_virtio16(_vq->vdev, vq->avail_flags_shadow);  in virtqueue_enable_cb_delayed()
885 	bufs = (u16)(vq->avail_idx_shadow - vq->last_used_idx) * 3 / 4;  in virtqueue_enable_cb_delayed()
887 	virtio_store_mb(vq->weak_barriers,  in virtqueue_enable_cb_delayed()
888 			&vring_used_event(&vq->vring),  in virtqueue_enable_cb_delayed()
889 			cpu_to_virtio16(_vq->vdev, vq->last_used_idx + bufs));  in virtqueue_enable_cb_delayed()
891 	if (unlikely((u16)(virtio16_to_cpu(_vq->vdev, vq->vring.used->idx) - vq->last_used_idx) > bufs)) {  in virtqueue_enable_cb_delayed()
892 		END_USE(vq);  in virtqueue_enable_cb_delayed()
896 	END_USE(vq);  in virtqueue_enable_cb_delayed()
911 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_detach_unused_buf()  local
915 	START_USE(vq);  in virtqueue_detach_unused_buf()
917 	for (i = 0; i < vq->vring.num; i++) {  in virtqueue_detach_unused_buf()
918 		if (!vq->desc_state[i].data)  in virtqueue_detach_unused_buf()
921 		buf = vq->desc_state[i].data;  in virtqueue_detach_unused_buf()
922 		detach_buf(vq, i, NULL);  in virtqueue_detach_unused_buf()
923 		vq->avail_idx_shadow--;  in virtqueue_detach_unused_buf()
924 		vq->vring.avail->idx = cpu_to_virtio16(_vq->vdev, vq->avail_idx_shadow);  in virtqueue_detach_unused_buf()
925 		END_USE(vq);  in virtqueue_detach_unused_buf()
929 	BUG_ON(vq->vq.num_free != vq->vring.num);  in virtqueue_detach_unused_buf()
931 	END_USE(vq);  in virtqueue_detach_unused_buf()
938 	struct vring_virtqueue *vq = to_vvq(_vq);  in vring_interrupt()  local
940 	if (!more_used(vq)) {  in vring_interrupt()
941 		pr_debug("virtqueue interrupt with no work for %p\n", vq);  in vring_interrupt()
945 	if (unlikely(vq->broken))  in vring_interrupt()
948 	pr_debug("virtqueue callback for %p (%p)\n", vq, vq->vq.callback);  in vring_interrupt()
949 	if (vq->vq.callback)  in vring_interrupt()
950 		vq->vq.callback(&vq->vq);  in vring_interrupt()
966 	struct vring_virtqueue *vq;  in __vring_new_virtqueue()  local
968 	vq = kmalloc(sizeof(*vq) + vring.num * sizeof(struct vring_desc_state),  in __vring_new_virtqueue()
970 	if (!vq)  in __vring_new_virtqueue()
973 	vq->vring = vring;  in __vring_new_virtqueue()
974 	vq->vq.callback = callback;  in __vring_new_virtqueue()
975 	vq->vq.vdev = vdev;  in __vring_new_virtqueue()
976 	vq->vq.name = name;  in __vring_new_virtqueue()
977 	vq->vq.num_free = vring.num;  in __vring_new_virtqueue()
978 	vq->vq.index = index;  in __vring_new_virtqueue()
979 	vq->we_own_ring = false;  in __vring_new_virtqueue()
980 	vq->queue_dma_addr = 0;  in __vring_new_virtqueue()
981 	vq->queue_size_in_bytes = 0;  in __vring_new_virtqueue()
982 	vq->notify = notify;  in __vring_new_virtqueue()
983 	vq->weak_barriers = weak_barriers;  in __vring_new_virtqueue()
984 	vq->broken = false;  in __vring_new_virtqueue()
985 	vq->last_used_idx = 0;  in __vring_new_virtqueue()
986 	vq->avail_flags_shadow = 0;  in __vring_new_virtqueue()
987 	vq->avail_idx_shadow = 0;  in __vring_new_virtqueue()
988 	vq->num_added = 0;  in __vring_new_virtqueue()
989 	list_add_tail(&vq->vq.list, &vdev->vqs);  in __vring_new_virtqueue()
991 	vq->in_use = false;  in __vring_new_virtqueue()
992 	vq->last_add_time_valid = false;  in __vring_new_virtqueue()
995 	vq->indirect = virtio_has_feature(vdev, VIRTIO_RING_F_INDIRECT_DESC) &&  in __vring_new_virtqueue()
997 	vq->event = virtio_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX);  in __vring_new_virtqueue()
1001 		vq->avail_flags_shadow |= VRING_AVAIL_F_NO_INTERRUPT;  in __vring_new_virtqueue()
1002 		if (!vq->event)  in __vring_new_virtqueue()
1003 			vq->vring.avail->flags = cpu_to_virtio16(vdev, vq->avail_flags_shadow);  in __vring_new_virtqueue()
1007 	vq->free_head = 0;  in __vring_new_virtqueue()
1009 		vq->vring.desc[i].next = cpu_to_virtio16(vdev, i + 1);  in __vring_new_virtqueue()
1010 	memset(vq->desc_state, 0, vring.num * sizeof(struct vring_desc_state));  in __vring_new_virtqueue()
1012 	return &vq->vq;  in __vring_new_virtqueue()
1070 	struct virtqueue *vq;  in vring_create_virtqueue()  local
1105 	vq = __vring_new_virtqueue(index, vring, vdev, weak_barriers, context,  in vring_create_virtqueue()
1107 	if (!vq) {  in vring_create_virtqueue()
1113 	to_vvq(vq)->queue_dma_addr = dma_addr;  in vring_create_virtqueue()
1114 	to_vvq(vq)->queue_size_in_bytes = queue_size_in_bytes;  in vring_create_virtqueue()
1115 	to_vvq(vq)->we_own_ring = true;  in vring_create_virtqueue()
1117 	return vq;  in vring_create_virtqueue()
1128 				      bool (*notify)(struct virtqueue *vq),  in vring_new_virtqueue()  argument
1129 				      void (*callback)(struct virtqueue *vq),  in vring_new_virtqueue()  argument
1141 	struct vring_virtqueue *vq = to_vvq(_vq);  in vring_del_virtqueue()  local
1143 	if (vq->we_own_ring) {  in vring_del_virtqueue()
1144 		vring_free_queue(vq->vq.vdev, vq->queue_size_in_bytes,  in vring_del_virtqueue()
1145 				 vq->vring.desc, vq->queue_dma_addr);  in vring_del_virtqueue()
1148 	kfree(vq);  in vring_del_virtqueue()
1185 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_get_vring_size()  local
1187 	return vq->vring.num;  in virtqueue_get_vring_size()
1193 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_is_broken()  local
1195 	return vq->broken;  in virtqueue_is_broken()
1208 		struct vring_virtqueue *vq = to_vvq(_vq);  in virtio_break_device()  local
1209 		vq->broken = true;  in virtio_break_device()
1216 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_get_desc_addr()  local
1218 	BUG_ON(!vq->we_own_ring);  in virtqueue_get_desc_addr()
1220 	return vq->queue_dma_addr;  in virtqueue_get_desc_addr()
1226 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_get_avail_addr()  local
1228 	BUG_ON(!vq->we_own_ring);  in virtqueue_get_avail_addr()
1230 	return vq->queue_dma_addr +  in virtqueue_get_avail_addr()
1231 		((char *)vq->vring.avail - (char *)vq->vring.desc);  in virtqueue_get_avail_addr()
1237 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_get_used_addr()  local
1239 	BUG_ON(!vq->we_own_ring);  in virtqueue_get_used_addr()
1241 	return vq->queue_dma_addr +  in virtqueue_get_used_addr()
1242 		((char *)vq->vring.used - (char *)vq->vring.desc);  in virtqueue_get_used_addr()
1246 const struct vring *virtqueue_get_vring(struct virtqueue *vq)  in virtqueue_get_vring()  argument
1248 	return &to_vvq(vq)->vring;  in virtqueue_get_vring()