Lines Matching refs:vq
20 dev_err(&(_vq)->vq.vdev->dev, \
21 "%s:"fmt, (_vq)->vq.name, ##args); \
29 (_vq)->vq.name, (_vq)->in_use); \
57 dev_err(&_vq->vq.vdev->dev, \
58 "%s:"fmt, (_vq)->vq.name, ##args); \
61 #define START_USE(vq) argument
62 #define END_USE(vq) argument
63 #define LAST_ADD_TIME_UPDATE(vq) argument
64 #define LAST_ADD_TIME_CHECK(vq) argument
65 #define LAST_ADD_TIME_INVALID(vq) argument
88 struct virtqueue vq; member
181 bool (*notify)(struct virtqueue *vq);
201 #define to_vvq(_vq) container_of(_vq, struct vring_virtqueue, vq)
206 struct vring_virtqueue *vq = to_vvq(_vq); in virtqueue_use_indirect() local
212 return (vq->indirect && total_sg > 1 && vq->vq.num_free); in virtqueue_use_indirect()
319 static inline struct device *vring_dma_dev(const struct vring_virtqueue *vq) in vring_dma_dev() argument
321 return vq->vq.vdev->dev.parent; in vring_dma_dev()
325 static dma_addr_t vring_map_one_sg(const struct vring_virtqueue *vq, in vring_map_one_sg() argument
329 if (!vq->use_dma_api) in vring_map_one_sg()
337 return dma_map_page(vring_dma_dev(vq), in vring_map_one_sg()
342 static dma_addr_t vring_map_single(const struct vring_virtqueue *vq, in vring_map_single() argument
346 if (!vq->use_dma_api) in vring_map_single()
349 return dma_map_single(vring_dma_dev(vq), in vring_map_single()
353 static int vring_mapping_error(const struct vring_virtqueue *vq, in vring_mapping_error() argument
356 if (!vq->use_dma_api) in vring_mapping_error()
359 return dma_mapping_error(vring_dma_dev(vq), addr); in vring_mapping_error()
367 static void vring_unmap_one_split(const struct vring_virtqueue *vq, in vring_unmap_one_split() argument
372 if (!vq->use_dma_api) in vring_unmap_one_split()
375 flags = virtio16_to_cpu(vq->vq.vdev, desc->flags); in vring_unmap_one_split()
378 dma_unmap_single(vring_dma_dev(vq), in vring_unmap_one_split()
379 virtio64_to_cpu(vq->vq.vdev, desc->addr), in vring_unmap_one_split()
380 virtio32_to_cpu(vq->vq.vdev, desc->len), in vring_unmap_one_split()
384 dma_unmap_page(vring_dma_dev(vq), in vring_unmap_one_split()
385 virtio64_to_cpu(vq->vq.vdev, desc->addr), in vring_unmap_one_split()
386 virtio32_to_cpu(vq->vq.vdev, desc->len), in vring_unmap_one_split()
424 struct vring_virtqueue *vq = to_vvq(_vq); in virtqueue_add_split() local
431 START_USE(vq); in virtqueue_add_split()
434 BUG_ON(ctx && vq->indirect); in virtqueue_add_split()
436 if (unlikely(vq->broken)) { in virtqueue_add_split()
437 END_USE(vq); in virtqueue_add_split()
441 LAST_ADD_TIME_UPDATE(vq); in virtqueue_add_split()
445 head = vq->free_head; in virtqueue_add_split()
451 WARN_ON_ONCE(total_sg > vq->split.vring.num && !vq->indirect); in virtqueue_add_split()
462 desc = vq->split.vring.desc; in virtqueue_add_split()
467 if (vq->vq.num_free < descs_used) { in virtqueue_add_split()
469 descs_used, vq->vq.num_free); in virtqueue_add_split()
474 vq->notify(&vq->vq); in virtqueue_add_split()
477 END_USE(vq); in virtqueue_add_split()
483 dma_addr_t addr = vring_map_one_sg(vq, sg, DMA_TO_DEVICE); in virtqueue_add_split()
484 if (vring_mapping_error(vq, addr)) in virtqueue_add_split()
496 dma_addr_t addr = vring_map_one_sg(vq, sg, DMA_FROM_DEVICE); in virtqueue_add_split()
497 if (vring_mapping_error(vq, addr)) in virtqueue_add_split()
513 vq, desc, total_sg * sizeof(struct vring_desc), in virtqueue_add_split()
515 if (vring_mapping_error(vq, addr)) in virtqueue_add_split()
518 vq->split.vring.desc[head].flags = cpu_to_virtio16(_vq->vdev, in virtqueue_add_split()
520 vq->split.vring.desc[head].addr = cpu_to_virtio64(_vq->vdev, in virtqueue_add_split()
523 vq->split.vring.desc[head].len = cpu_to_virtio32(_vq->vdev, in virtqueue_add_split()
528 vq->vq.num_free -= descs_used; in virtqueue_add_split()
532 vq->free_head = virtio16_to_cpu(_vq->vdev, in virtqueue_add_split()
533 vq->split.vring.desc[head].next); in virtqueue_add_split()
535 vq->free_head = i; in virtqueue_add_split()
538 vq->split.desc_state[head].data = data; in virtqueue_add_split()
540 vq->split.desc_state[head].indir_desc = desc; in virtqueue_add_split()
542 vq->split.desc_state[head].indir_desc = ctx; in virtqueue_add_split()
546 avail = vq->split.avail_idx_shadow & (vq->split.vring.num - 1); in virtqueue_add_split()
547 vq->split.vring.avail->ring[avail] = cpu_to_virtio16(_vq->vdev, head); in virtqueue_add_split()
551 virtio_wmb(vq->weak_barriers); in virtqueue_add_split()
552 vq->split.avail_idx_shadow++; in virtqueue_add_split()
553 vq->split.vring.avail->idx = cpu_to_virtio16(_vq->vdev, in virtqueue_add_split()
554 vq->split.avail_idx_shadow); in virtqueue_add_split()
555 vq->num_added++; in virtqueue_add_split()
557 pr_debug("Added buffer head %i to %p\n", head, vq); in virtqueue_add_split()
558 END_USE(vq); in virtqueue_add_split()
562 if (unlikely(vq->num_added == (1 << 16) - 1)) in virtqueue_add_split()
578 vring_unmap_one_split(vq, &desc[i]); in virtqueue_add_split()
585 END_USE(vq); in virtqueue_add_split()
591 struct vring_virtqueue *vq = to_vvq(_vq); in virtqueue_kick_prepare_split() local
595 START_USE(vq); in virtqueue_kick_prepare_split()
598 virtio_mb(vq->weak_barriers); in virtqueue_kick_prepare_split()
600 old = vq->split.avail_idx_shadow - vq->num_added; in virtqueue_kick_prepare_split()
601 new = vq->split.avail_idx_shadow; in virtqueue_kick_prepare_split()
602 vq->num_added = 0; in virtqueue_kick_prepare_split()
604 LAST_ADD_TIME_CHECK(vq); in virtqueue_kick_prepare_split()
605 LAST_ADD_TIME_INVALID(vq); in virtqueue_kick_prepare_split()
607 if (vq->event) { in virtqueue_kick_prepare_split()
609 vring_avail_event(&vq->split.vring)), in virtqueue_kick_prepare_split()
612 needs_kick = !(vq->split.vring.used->flags & in virtqueue_kick_prepare_split()
616 END_USE(vq); in virtqueue_kick_prepare_split()
620 static void detach_buf_split(struct vring_virtqueue *vq, unsigned int head, in detach_buf_split() argument
624 __virtio16 nextflag = cpu_to_virtio16(vq->vq.vdev, VRING_DESC_F_NEXT); in detach_buf_split()
627 vq->split.desc_state[head].data = NULL; in detach_buf_split()
632 while (vq->split.vring.desc[i].flags & nextflag) { in detach_buf_split()
633 vring_unmap_one_split(vq, &vq->split.vring.desc[i]); in detach_buf_split()
634 i = virtio16_to_cpu(vq->vq.vdev, vq->split.vring.desc[i].next); in detach_buf_split()
635 vq->vq.num_free++; in detach_buf_split()
638 vring_unmap_one_split(vq, &vq->split.vring.desc[i]); in detach_buf_split()
639 vq->split.vring.desc[i].next = cpu_to_virtio16(vq->vq.vdev, in detach_buf_split()
640 vq->free_head); in detach_buf_split()
641 vq->free_head = head; in detach_buf_split()
644 vq->vq.num_free++; in detach_buf_split()
646 if (vq->indirect) { in detach_buf_split()
648 vq->split.desc_state[head].indir_desc; in detach_buf_split()
655 len = virtio32_to_cpu(vq->vq.vdev, in detach_buf_split()
656 vq->split.vring.desc[head].len); in detach_buf_split()
658 BUG_ON(!(vq->split.vring.desc[head].flags & in detach_buf_split()
659 cpu_to_virtio16(vq->vq.vdev, VRING_DESC_F_INDIRECT))); in detach_buf_split()
663 vring_unmap_one_split(vq, &indir_desc[j]); in detach_buf_split()
666 vq->split.desc_state[head].indir_desc = NULL; in detach_buf_split()
668 *ctx = vq->split.desc_state[head].indir_desc; in detach_buf_split()
672 static inline bool more_used_split(const struct vring_virtqueue *vq) in more_used_split() argument
674 return vq->last_used_idx != virtio16_to_cpu(vq->vq.vdev, in more_used_split()
675 vq->split.vring.used->idx); in more_used_split()
682 struct vring_virtqueue *vq = to_vvq(_vq); in virtqueue_get_buf_ctx_split() local
687 START_USE(vq); in virtqueue_get_buf_ctx_split()
689 if (unlikely(vq->broken)) { in virtqueue_get_buf_ctx_split()
690 END_USE(vq); in virtqueue_get_buf_ctx_split()
694 if (!more_used_split(vq)) { in virtqueue_get_buf_ctx_split()
696 END_USE(vq); in virtqueue_get_buf_ctx_split()
701 virtio_rmb(vq->weak_barriers); in virtqueue_get_buf_ctx_split()
703 last_used = (vq->last_used_idx & (vq->split.vring.num - 1)); in virtqueue_get_buf_ctx_split()
705 vq->split.vring.used->ring[last_used].id); in virtqueue_get_buf_ctx_split()
707 vq->split.vring.used->ring[last_used].len); in virtqueue_get_buf_ctx_split()
709 if (unlikely(i >= vq->split.vring.num)) { in virtqueue_get_buf_ctx_split()
710 BAD_RING(vq, "id %u out of range\n", i); in virtqueue_get_buf_ctx_split()
713 if (unlikely(!vq->split.desc_state[i].data)) { in virtqueue_get_buf_ctx_split()
714 BAD_RING(vq, "id %u is not a head!\n", i); in virtqueue_get_buf_ctx_split()
719 ret = vq->split.desc_state[i].data; in virtqueue_get_buf_ctx_split()
720 detach_buf_split(vq, i, ctx); in virtqueue_get_buf_ctx_split()
721 vq->last_used_idx++; in virtqueue_get_buf_ctx_split()
725 if (!(vq->split.avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT)) in virtqueue_get_buf_ctx_split()
726 virtio_store_mb(vq->weak_barriers, in virtqueue_get_buf_ctx_split()
727 &vring_used_event(&vq->split.vring), in virtqueue_get_buf_ctx_split()
728 cpu_to_virtio16(_vq->vdev, vq->last_used_idx)); in virtqueue_get_buf_ctx_split()
730 LAST_ADD_TIME_INVALID(vq); in virtqueue_get_buf_ctx_split()
732 END_USE(vq); in virtqueue_get_buf_ctx_split()
738 struct vring_virtqueue *vq = to_vvq(_vq); in virtqueue_disable_cb_split() local
740 if (!(vq->split.avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT)) { in virtqueue_disable_cb_split()
741 vq->split.avail_flags_shadow |= VRING_AVAIL_F_NO_INTERRUPT; in virtqueue_disable_cb_split()
742 if (!vq->event) in virtqueue_disable_cb_split()
743 vq->split.vring.avail->flags = in virtqueue_disable_cb_split()
745 vq->split.avail_flags_shadow); in virtqueue_disable_cb_split()
751 struct vring_virtqueue *vq = to_vvq(_vq); in virtqueue_enable_cb_prepare_split() local
754 START_USE(vq); in virtqueue_enable_cb_prepare_split()
761 if (vq->split.avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT) { in virtqueue_enable_cb_prepare_split()
762 vq->split.avail_flags_shadow &= ~VRING_AVAIL_F_NO_INTERRUPT; in virtqueue_enable_cb_prepare_split()
763 if (!vq->event) in virtqueue_enable_cb_prepare_split()
764 vq->split.vring.avail->flags = in virtqueue_enable_cb_prepare_split()
766 vq->split.avail_flags_shadow); in virtqueue_enable_cb_prepare_split()
768 vring_used_event(&vq->split.vring) = cpu_to_virtio16(_vq->vdev, in virtqueue_enable_cb_prepare_split()
769 last_used_idx = vq->last_used_idx); in virtqueue_enable_cb_prepare_split()
770 END_USE(vq); in virtqueue_enable_cb_prepare_split()
776 struct vring_virtqueue *vq = to_vvq(_vq); in virtqueue_poll_split() local
779 vq->split.vring.used->idx); in virtqueue_poll_split()
784 struct vring_virtqueue *vq = to_vvq(_vq); in virtqueue_enable_cb_delayed_split() local
787 START_USE(vq); in virtqueue_enable_cb_delayed_split()
794 if (vq->split.avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT) { in virtqueue_enable_cb_delayed_split()
795 vq->split.avail_flags_shadow &= ~VRING_AVAIL_F_NO_INTERRUPT; in virtqueue_enable_cb_delayed_split()
796 if (!vq->event) in virtqueue_enable_cb_delayed_split()
797 vq->split.vring.avail->flags = in virtqueue_enable_cb_delayed_split()
799 vq->split.avail_flags_shadow); in virtqueue_enable_cb_delayed_split()
802 bufs = (u16)(vq->split.avail_idx_shadow - vq->last_used_idx) * 3 / 4; in virtqueue_enable_cb_delayed_split()
804 virtio_store_mb(vq->weak_barriers, in virtqueue_enable_cb_delayed_split()
805 &vring_used_event(&vq->split.vring), in virtqueue_enable_cb_delayed_split()
806 cpu_to_virtio16(_vq->vdev, vq->last_used_idx + bufs)); in virtqueue_enable_cb_delayed_split()
808 if (unlikely((u16)(virtio16_to_cpu(_vq->vdev, vq->split.vring.used->idx) in virtqueue_enable_cb_delayed_split()
809 - vq->last_used_idx) > bufs)) { in virtqueue_enable_cb_delayed_split()
810 END_USE(vq); in virtqueue_enable_cb_delayed_split()
814 END_USE(vq); in virtqueue_enable_cb_delayed_split()
820 struct vring_virtqueue *vq = to_vvq(_vq); in virtqueue_detach_unused_buf_split() local
824 START_USE(vq); in virtqueue_detach_unused_buf_split()
826 for (i = 0; i < vq->split.vring.num; i++) { in virtqueue_detach_unused_buf_split()
827 if (!vq->split.desc_state[i].data) in virtqueue_detach_unused_buf_split()
830 buf = vq->split.desc_state[i].data; in virtqueue_detach_unused_buf_split()
831 detach_buf_split(vq, i, NULL); in virtqueue_detach_unused_buf_split()
832 vq->split.avail_idx_shadow--; in virtqueue_detach_unused_buf_split()
833 vq->split.vring.avail->idx = cpu_to_virtio16(_vq->vdev, in virtqueue_detach_unused_buf_split()
834 vq->split.avail_idx_shadow); in virtqueue_detach_unused_buf_split()
835 END_USE(vq); in virtqueue_detach_unused_buf_split()
839 BUG_ON(vq->vq.num_free != vq->split.vring.num); in virtqueue_detach_unused_buf_split()
841 END_USE(vq); in virtqueue_detach_unused_buf_split()
857 struct virtqueue *vq; in vring_create_virtqueue_split() local
894 vq = __vring_new_virtqueue(index, vring, vdev, weak_barriers, context, in vring_create_virtqueue_split()
896 if (!vq) { in vring_create_virtqueue_split()
902 to_vvq(vq)->split.queue_dma_addr = dma_addr; in vring_create_virtqueue_split()
903 to_vvq(vq)->split.queue_size_in_bytes = queue_size_in_bytes; in vring_create_virtqueue_split()
904 to_vvq(vq)->we_own_ring = true; in vring_create_virtqueue_split()
906 return vq; in vring_create_virtqueue_split()
914 static void vring_unmap_state_packed(const struct vring_virtqueue *vq, in vring_unmap_state_packed() argument
919 if (!vq->use_dma_api) in vring_unmap_state_packed()
925 dma_unmap_single(vring_dma_dev(vq), in vring_unmap_state_packed()
930 dma_unmap_page(vring_dma_dev(vq), in vring_unmap_state_packed()
937 static void vring_unmap_desc_packed(const struct vring_virtqueue *vq, in vring_unmap_desc_packed() argument
942 if (!vq->use_dma_api) in vring_unmap_desc_packed()
948 dma_unmap_single(vring_dma_dev(vq), in vring_unmap_desc_packed()
954 dma_unmap_page(vring_dma_dev(vq), in vring_unmap_desc_packed()
979 static int virtqueue_add_indirect_packed(struct vring_virtqueue *vq, in virtqueue_add_indirect_packed() argument
993 head = vq->packed.next_avail_idx; in virtqueue_add_indirect_packed()
996 if (unlikely(vq->vq.num_free < 1)) { in virtqueue_add_indirect_packed()
999 END_USE(vq); in virtqueue_add_indirect_packed()
1004 id = vq->free_head; in virtqueue_add_indirect_packed()
1005 BUG_ON(id == vq->packed.vring.num); in virtqueue_add_indirect_packed()
1009 addr = vring_map_one_sg(vq, sg, n < out_sgs ? in virtqueue_add_indirect_packed()
1011 if (vring_mapping_error(vq, addr)) in virtqueue_add_indirect_packed()
1023 addr = vring_map_single(vq, desc, in virtqueue_add_indirect_packed()
1026 if (vring_mapping_error(vq, addr)) in virtqueue_add_indirect_packed()
1029 vq->packed.vring.desc[head].addr = cpu_to_le64(addr); in virtqueue_add_indirect_packed()
1030 vq->packed.vring.desc[head].len = cpu_to_le32(total_sg * in virtqueue_add_indirect_packed()
1032 vq->packed.vring.desc[head].id = cpu_to_le16(id); in virtqueue_add_indirect_packed()
1034 if (vq->use_dma_api) { in virtqueue_add_indirect_packed()
1035 vq->packed.desc_extra[id].addr = addr; in virtqueue_add_indirect_packed()
1036 vq->packed.desc_extra[id].len = total_sg * in virtqueue_add_indirect_packed()
1038 vq->packed.desc_extra[id].flags = VRING_DESC_F_INDIRECT | in virtqueue_add_indirect_packed()
1039 vq->packed.avail_used_flags; in virtqueue_add_indirect_packed()
1047 virtio_wmb(vq->weak_barriers); in virtqueue_add_indirect_packed()
1048 vq->packed.vring.desc[head].flags = cpu_to_le16(VRING_DESC_F_INDIRECT | in virtqueue_add_indirect_packed()
1049 vq->packed.avail_used_flags); in virtqueue_add_indirect_packed()
1052 vq->vq.num_free -= 1; in virtqueue_add_indirect_packed()
1056 if (n >= vq->packed.vring.num) { in virtqueue_add_indirect_packed()
1058 vq->packed.avail_wrap_counter ^= 1; in virtqueue_add_indirect_packed()
1059 vq->packed.avail_used_flags ^= in virtqueue_add_indirect_packed()
1063 vq->packed.next_avail_idx = n; in virtqueue_add_indirect_packed()
1064 vq->free_head = vq->packed.desc_state[id].next; in virtqueue_add_indirect_packed()
1067 vq->packed.desc_state[id].num = 1; in virtqueue_add_indirect_packed()
1068 vq->packed.desc_state[id].data = data; in virtqueue_add_indirect_packed()
1069 vq->packed.desc_state[id].indir_desc = desc; in virtqueue_add_indirect_packed()
1070 vq->packed.desc_state[id].last = id; in virtqueue_add_indirect_packed()
1072 vq->num_added += 1; in virtqueue_add_indirect_packed()
1074 pr_debug("Added buffer head %i to %p\n", head, vq); in virtqueue_add_indirect_packed()
1075 END_USE(vq); in virtqueue_add_indirect_packed()
1083 vring_unmap_desc_packed(vq, &desc[i]); in virtqueue_add_indirect_packed()
1087 END_USE(vq); in virtqueue_add_indirect_packed()
1100 struct vring_virtqueue *vq = to_vvq(_vq); in virtqueue_add_packed() local
1107 START_USE(vq); in virtqueue_add_packed()
1110 BUG_ON(ctx && vq->indirect); in virtqueue_add_packed()
1112 if (unlikely(vq->broken)) { in virtqueue_add_packed()
1113 END_USE(vq); in virtqueue_add_packed()
1117 LAST_ADD_TIME_UPDATE(vq); in virtqueue_add_packed()
1122 return virtqueue_add_indirect_packed(vq, sgs, total_sg, in virtqueue_add_packed()
1125 head = vq->packed.next_avail_idx; in virtqueue_add_packed()
1126 avail_used_flags = vq->packed.avail_used_flags; in virtqueue_add_packed()
1128 WARN_ON_ONCE(total_sg > vq->packed.vring.num && !vq->indirect); in virtqueue_add_packed()
1130 desc = vq->packed.vring.desc; in virtqueue_add_packed()
1134 if (unlikely(vq->vq.num_free < descs_used)) { in virtqueue_add_packed()
1136 descs_used, vq->vq.num_free); in virtqueue_add_packed()
1137 END_USE(vq); in virtqueue_add_packed()
1141 id = vq->free_head; in virtqueue_add_packed()
1142 BUG_ON(id == vq->packed.vring.num); in virtqueue_add_packed()
1148 dma_addr_t addr = vring_map_one_sg(vq, sg, n < out_sgs ? in virtqueue_add_packed()
1150 if (vring_mapping_error(vq, addr)) in virtqueue_add_packed()
1153 flags = cpu_to_le16(vq->packed.avail_used_flags | in virtqueue_add_packed()
1165 if (unlikely(vq->use_dma_api)) { in virtqueue_add_packed()
1166 vq->packed.desc_extra[curr].addr = addr; in virtqueue_add_packed()
1167 vq->packed.desc_extra[curr].len = sg->length; in virtqueue_add_packed()
1168 vq->packed.desc_extra[curr].flags = in virtqueue_add_packed()
1172 curr = vq->packed.desc_state[curr].next; in virtqueue_add_packed()
1174 if ((unlikely(++i >= vq->packed.vring.num))) { in virtqueue_add_packed()
1176 vq->packed.avail_used_flags ^= in virtqueue_add_packed()
1184 vq->packed.avail_wrap_counter ^= 1; in virtqueue_add_packed()
1187 vq->vq.num_free -= descs_used; in virtqueue_add_packed()
1190 vq->packed.next_avail_idx = i; in virtqueue_add_packed()
1191 vq->free_head = curr; in virtqueue_add_packed()
1194 vq->packed.desc_state[id].num = descs_used; in virtqueue_add_packed()
1195 vq->packed.desc_state[id].data = data; in virtqueue_add_packed()
1196 vq->packed.desc_state[id].indir_desc = ctx; in virtqueue_add_packed()
1197 vq->packed.desc_state[id].last = prev; in virtqueue_add_packed()
1204 virtio_wmb(vq->weak_barriers); in virtqueue_add_packed()
1205 vq->packed.vring.desc[head].flags = head_flags; in virtqueue_add_packed()
1206 vq->num_added += descs_used; in virtqueue_add_packed()
1208 pr_debug("Added buffer head %i to %p\n", head, vq); in virtqueue_add_packed()
1209 END_USE(vq); in virtqueue_add_packed()
1217 vq->packed.avail_used_flags = avail_used_flags; in virtqueue_add_packed()
1222 vring_unmap_desc_packed(vq, &desc[i]); in virtqueue_add_packed()
1224 if (i >= vq->packed.vring.num) in virtqueue_add_packed()
1228 END_USE(vq); in virtqueue_add_packed()
1234 struct vring_virtqueue *vq = to_vvq(_vq); in virtqueue_kick_prepare_packed() local
1245 START_USE(vq); in virtqueue_kick_prepare_packed()
1251 virtio_mb(vq->weak_barriers); in virtqueue_kick_prepare_packed()
1253 old = vq->packed.next_avail_idx - vq->num_added; in virtqueue_kick_prepare_packed()
1254 new = vq->packed.next_avail_idx; in virtqueue_kick_prepare_packed()
1255 vq->num_added = 0; in virtqueue_kick_prepare_packed()
1257 snapshot.u32 = *(u32 *)vq->packed.vring.device; in virtqueue_kick_prepare_packed()
1260 LAST_ADD_TIME_CHECK(vq); in virtqueue_kick_prepare_packed()
1261 LAST_ADD_TIME_INVALID(vq); in virtqueue_kick_prepare_packed()
1272 if (wrap_counter != vq->packed.avail_wrap_counter) in virtqueue_kick_prepare_packed()
1273 event_idx -= vq->packed.vring.num; in virtqueue_kick_prepare_packed()
1277 END_USE(vq); in virtqueue_kick_prepare_packed()
1281 static void detach_buf_packed(struct vring_virtqueue *vq, in detach_buf_packed() argument
1288 state = &vq->packed.desc_state[id]; in detach_buf_packed()
1293 vq->packed.desc_state[state->last].next = vq->free_head; in detach_buf_packed()
1294 vq->free_head = id; in detach_buf_packed()
1295 vq->vq.num_free += state->num; in detach_buf_packed()
1297 if (unlikely(vq->use_dma_api)) { in detach_buf_packed()
1300 vring_unmap_state_packed(vq, in detach_buf_packed()
1301 &vq->packed.desc_extra[curr]); in detach_buf_packed()
1302 curr = vq->packed.desc_state[curr].next; in detach_buf_packed()
1306 if (vq->indirect) { in detach_buf_packed()
1314 if (vq->use_dma_api) { in detach_buf_packed()
1315 len = vq->packed.desc_extra[id].len; in detach_buf_packed()
1318 vring_unmap_desc_packed(vq, &desc[i]); in detach_buf_packed()
1327 static inline bool is_used_desc_packed(const struct vring_virtqueue *vq, in is_used_desc_packed() argument
1333 flags = le16_to_cpu(vq->packed.vring.desc[idx].flags); in is_used_desc_packed()
1340 static inline bool more_used_packed(const struct vring_virtqueue *vq) in more_used_packed() argument
1342 return is_used_desc_packed(vq, vq->last_used_idx, in more_used_packed()
1343 vq->packed.used_wrap_counter); in more_used_packed()
1350 struct vring_virtqueue *vq = to_vvq(_vq); in virtqueue_get_buf_ctx_packed() local
1354 START_USE(vq); in virtqueue_get_buf_ctx_packed()
1356 if (unlikely(vq->broken)) { in virtqueue_get_buf_ctx_packed()
1357 END_USE(vq); in virtqueue_get_buf_ctx_packed()
1361 if (!more_used_packed(vq)) { in virtqueue_get_buf_ctx_packed()
1363 END_USE(vq); in virtqueue_get_buf_ctx_packed()
1368 virtio_rmb(vq->weak_barriers); in virtqueue_get_buf_ctx_packed()
1370 last_used = vq->last_used_idx; in virtqueue_get_buf_ctx_packed()
1371 id = le16_to_cpu(vq->packed.vring.desc[last_used].id); in virtqueue_get_buf_ctx_packed()
1372 *len = le32_to_cpu(vq->packed.vring.desc[last_used].len); in virtqueue_get_buf_ctx_packed()
1374 if (unlikely(id >= vq->packed.vring.num)) { in virtqueue_get_buf_ctx_packed()
1375 BAD_RING(vq, "id %u out of range\n", id); in virtqueue_get_buf_ctx_packed()
1378 if (unlikely(!vq->packed.desc_state[id].data)) { in virtqueue_get_buf_ctx_packed()
1379 BAD_RING(vq, "id %u is not a head!\n", id); in virtqueue_get_buf_ctx_packed()
1384 ret = vq->packed.desc_state[id].data; in virtqueue_get_buf_ctx_packed()
1385 detach_buf_packed(vq, id, ctx); in virtqueue_get_buf_ctx_packed()
1387 vq->last_used_idx += vq->packed.desc_state[id].num; in virtqueue_get_buf_ctx_packed()
1388 if (unlikely(vq->last_used_idx >= vq->packed.vring.num)) { in virtqueue_get_buf_ctx_packed()
1389 vq->last_used_idx -= vq->packed.vring.num; in virtqueue_get_buf_ctx_packed()
1390 vq->packed.used_wrap_counter ^= 1; in virtqueue_get_buf_ctx_packed()
1398 if (vq->packed.event_flags_shadow == VRING_PACKED_EVENT_FLAG_DESC) in virtqueue_get_buf_ctx_packed()
1399 virtio_store_mb(vq->weak_barriers, in virtqueue_get_buf_ctx_packed()
1400 &vq->packed.vring.driver->off_wrap, in virtqueue_get_buf_ctx_packed()
1401 cpu_to_le16(vq->last_used_idx | in virtqueue_get_buf_ctx_packed()
1402 (vq->packed.used_wrap_counter << in virtqueue_get_buf_ctx_packed()
1405 LAST_ADD_TIME_INVALID(vq); in virtqueue_get_buf_ctx_packed()
1407 END_USE(vq); in virtqueue_get_buf_ctx_packed()
1413 struct vring_virtqueue *vq = to_vvq(_vq); in virtqueue_disable_cb_packed() local
1415 if (vq->packed.event_flags_shadow != VRING_PACKED_EVENT_FLAG_DISABLE) { in virtqueue_disable_cb_packed()
1416 vq->packed.event_flags_shadow = VRING_PACKED_EVENT_FLAG_DISABLE; in virtqueue_disable_cb_packed()
1417 vq->packed.vring.driver->flags = in virtqueue_disable_cb_packed()
1418 cpu_to_le16(vq->packed.event_flags_shadow); in virtqueue_disable_cb_packed()
1424 struct vring_virtqueue *vq = to_vvq(_vq); in virtqueue_enable_cb_prepare_packed() local
1426 START_USE(vq); in virtqueue_enable_cb_prepare_packed()
1433 if (vq->event) { in virtqueue_enable_cb_prepare_packed()
1434 vq->packed.vring.driver->off_wrap = in virtqueue_enable_cb_prepare_packed()
1435 cpu_to_le16(vq->last_used_idx | in virtqueue_enable_cb_prepare_packed()
1436 (vq->packed.used_wrap_counter << in virtqueue_enable_cb_prepare_packed()
1442 virtio_wmb(vq->weak_barriers); in virtqueue_enable_cb_prepare_packed()
1445 if (vq->packed.event_flags_shadow == VRING_PACKED_EVENT_FLAG_DISABLE) { in virtqueue_enable_cb_prepare_packed()
1446 vq->packed.event_flags_shadow = vq->event ? in virtqueue_enable_cb_prepare_packed()
1449 vq->packed.vring.driver->flags = in virtqueue_enable_cb_prepare_packed()
1450 cpu_to_le16(vq->packed.event_flags_shadow); in virtqueue_enable_cb_prepare_packed()
1453 END_USE(vq); in virtqueue_enable_cb_prepare_packed()
1454 return vq->last_used_idx | ((u16)vq->packed.used_wrap_counter << in virtqueue_enable_cb_prepare_packed()
1460 struct vring_virtqueue *vq = to_vvq(_vq); in virtqueue_poll_packed() local
1467 return is_used_desc_packed(vq, used_idx, wrap_counter); in virtqueue_poll_packed()
1472 struct vring_virtqueue *vq = to_vvq(_vq); in virtqueue_enable_cb_delayed_packed() local
1476 START_USE(vq); in virtqueue_enable_cb_delayed_packed()
1483 if (vq->event) { in virtqueue_enable_cb_delayed_packed()
1485 bufs = (vq->packed.vring.num - vq->vq.num_free) * 3 / 4; in virtqueue_enable_cb_delayed_packed()
1486 wrap_counter = vq->packed.used_wrap_counter; in virtqueue_enable_cb_delayed_packed()
1488 used_idx = vq->last_used_idx + bufs; in virtqueue_enable_cb_delayed_packed()
1489 if (used_idx >= vq->packed.vring.num) { in virtqueue_enable_cb_delayed_packed()
1490 used_idx -= vq->packed.vring.num; in virtqueue_enable_cb_delayed_packed()
1494 vq->packed.vring.driver->off_wrap = cpu_to_le16(used_idx | in virtqueue_enable_cb_delayed_packed()
1501 virtio_wmb(vq->weak_barriers); in virtqueue_enable_cb_delayed_packed()
1504 if (vq->packed.event_flags_shadow == VRING_PACKED_EVENT_FLAG_DISABLE) { in virtqueue_enable_cb_delayed_packed()
1505 vq->packed.event_flags_shadow = vq->event ? in virtqueue_enable_cb_delayed_packed()
1508 vq->packed.vring.driver->flags = in virtqueue_enable_cb_delayed_packed()
1509 cpu_to_le16(vq->packed.event_flags_shadow); in virtqueue_enable_cb_delayed_packed()
1516 virtio_mb(vq->weak_barriers); in virtqueue_enable_cb_delayed_packed()
1518 if (is_used_desc_packed(vq, in virtqueue_enable_cb_delayed_packed()
1519 vq->last_used_idx, in virtqueue_enable_cb_delayed_packed()
1520 vq->packed.used_wrap_counter)) { in virtqueue_enable_cb_delayed_packed()
1521 END_USE(vq); in virtqueue_enable_cb_delayed_packed()
1525 END_USE(vq); in virtqueue_enable_cb_delayed_packed()
1531 struct vring_virtqueue *vq = to_vvq(_vq); in virtqueue_detach_unused_buf_packed() local
1535 START_USE(vq); in virtqueue_detach_unused_buf_packed()
1537 for (i = 0; i < vq->packed.vring.num; i++) { in virtqueue_detach_unused_buf_packed()
1538 if (!vq->packed.desc_state[i].data) in virtqueue_detach_unused_buf_packed()
1541 buf = vq->packed.desc_state[i].data; in virtqueue_detach_unused_buf_packed()
1542 detach_buf_packed(vq, i, NULL); in virtqueue_detach_unused_buf_packed()
1543 END_USE(vq); in virtqueue_detach_unused_buf_packed()
1547 BUG_ON(vq->vq.num_free != vq->packed.vring.num); in virtqueue_detach_unused_buf_packed()
1549 END_USE(vq); in virtqueue_detach_unused_buf_packed()
1565 struct vring_virtqueue *vq; in vring_create_virtqueue_packed() local
1594 vq = kmalloc(sizeof(*vq), GFP_KERNEL); in vring_create_virtqueue_packed()
1595 if (!vq) in vring_create_virtqueue_packed()
1598 vq->vq.callback = callback; in vring_create_virtqueue_packed()
1599 vq->vq.vdev = vdev; in vring_create_virtqueue_packed()
1600 vq->vq.name = name; in vring_create_virtqueue_packed()
1601 vq->vq.num_free = num; in vring_create_virtqueue_packed()
1602 vq->vq.index = index; in vring_create_virtqueue_packed()
1603 vq->we_own_ring = true; in vring_create_virtqueue_packed()
1604 vq->notify = notify; in vring_create_virtqueue_packed()
1605 vq->weak_barriers = weak_barriers; in vring_create_virtqueue_packed()
1606 vq->broken = false; in vring_create_virtqueue_packed()
1607 vq->last_used_idx = 0; in vring_create_virtqueue_packed()
1608 vq->num_added = 0; in vring_create_virtqueue_packed()
1609 vq->packed_ring = true; in vring_create_virtqueue_packed()
1610 vq->use_dma_api = vring_use_dma_api(vdev); in vring_create_virtqueue_packed()
1611 list_add_tail(&vq->vq.list, &vdev->vqs); in vring_create_virtqueue_packed()
1613 vq->in_use = false; in vring_create_virtqueue_packed()
1614 vq->last_add_time_valid = false; in vring_create_virtqueue_packed()
1617 vq->indirect = virtio_has_feature(vdev, VIRTIO_RING_F_INDIRECT_DESC) && in vring_create_virtqueue_packed()
1619 vq->event = virtio_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX); in vring_create_virtqueue_packed()
1622 vq->weak_barriers = false; in vring_create_virtqueue_packed()
1624 vq->packed.ring_dma_addr = ring_dma_addr; in vring_create_virtqueue_packed()
1625 vq->packed.driver_event_dma_addr = driver_event_dma_addr; in vring_create_virtqueue_packed()
1626 vq->packed.device_event_dma_addr = device_event_dma_addr; in vring_create_virtqueue_packed()
1628 vq->packed.ring_size_in_bytes = ring_size_in_bytes; in vring_create_virtqueue_packed()
1629 vq->packed.event_size_in_bytes = event_size_in_bytes; in vring_create_virtqueue_packed()
1631 vq->packed.vring.num = num; in vring_create_virtqueue_packed()
1632 vq->packed.vring.desc = ring; in vring_create_virtqueue_packed()
1633 vq->packed.vring.driver = driver; in vring_create_virtqueue_packed()
1634 vq->packed.vring.device = device; in vring_create_virtqueue_packed()
1636 vq->packed.next_avail_idx = 0; in vring_create_virtqueue_packed()
1637 vq->packed.avail_wrap_counter = 1; in vring_create_virtqueue_packed()
1638 vq->packed.used_wrap_counter = 1; in vring_create_virtqueue_packed()
1639 vq->packed.event_flags_shadow = 0; in vring_create_virtqueue_packed()
1640 vq->packed.avail_used_flags = 1 << VRING_PACKED_DESC_F_AVAIL; in vring_create_virtqueue_packed()
1642 vq->packed.desc_state = kmalloc_array(num, in vring_create_virtqueue_packed()
1645 if (!vq->packed.desc_state) in vring_create_virtqueue_packed()
1648 memset(vq->packed.desc_state, 0, in vring_create_virtqueue_packed()
1652 vq->free_head = 0; in vring_create_virtqueue_packed()
1654 vq->packed.desc_state[i].next = i + 1; in vring_create_virtqueue_packed()
1656 vq->packed.desc_extra = kmalloc_array(num, in vring_create_virtqueue_packed()
1659 if (!vq->packed.desc_extra) in vring_create_virtqueue_packed()
1662 memset(vq->packed.desc_extra, 0, in vring_create_virtqueue_packed()
1667 vq->packed.event_flags_shadow = VRING_PACKED_EVENT_FLAG_DISABLE; in vring_create_virtqueue_packed()
1668 vq->packed.vring.driver->flags = in vring_create_virtqueue_packed()
1669 cpu_to_le16(vq->packed.event_flags_shadow); in vring_create_virtqueue_packed()
1672 return &vq->vq; in vring_create_virtqueue_packed()
1675 kfree(vq->packed.desc_state); in vring_create_virtqueue_packed()
1677 kfree(vq); in vring_create_virtqueue_packed()
1702 struct vring_virtqueue *vq = to_vvq(_vq); in virtqueue_add() local
1704 return vq->packed_ring ? virtqueue_add_packed(_vq, sgs, total_sg, in virtqueue_add()
1758 int virtqueue_add_outbuf(struct virtqueue *vq, in virtqueue_add_outbuf() argument
1763 return virtqueue_add(vq, &sg, num, 1, 0, data, NULL, gfp); in virtqueue_add_outbuf()
1780 int virtqueue_add_inbuf(struct virtqueue *vq, in virtqueue_add_inbuf() argument
1785 return virtqueue_add(vq, &sg, num, 0, 1, data, NULL, gfp); in virtqueue_add_inbuf()
1803 int virtqueue_add_inbuf_ctx(struct virtqueue *vq, in virtqueue_add_inbuf_ctx() argument
1809 return virtqueue_add(vq, &sg, num, 0, 1, data, ctx, gfp); in virtqueue_add_inbuf_ctx()
1826 struct vring_virtqueue *vq = to_vvq(_vq); in virtqueue_kick_prepare() local
1828 return vq->packed_ring ? virtqueue_kick_prepare_packed(_vq) : in virtqueue_kick_prepare()
1843 struct vring_virtqueue *vq = to_vvq(_vq); in virtqueue_notify() local
1845 if (unlikely(vq->broken)) in virtqueue_notify()
1849 if (!vq->notify(_vq)) { in virtqueue_notify()
1850 vq->broken = true; in virtqueue_notify()
1869 bool virtqueue_kick(struct virtqueue *vq) in virtqueue_kick() argument
1871 if (virtqueue_kick_prepare(vq)) in virtqueue_kick()
1872 return virtqueue_notify(vq); in virtqueue_kick()
1897 struct vring_virtqueue *vq = to_vvq(_vq); in virtqueue_get_buf_ctx() local
1899 return vq->packed_ring ? virtqueue_get_buf_ctx_packed(_vq, len, ctx) : in virtqueue_get_buf_ctx()
1920 struct vring_virtqueue *vq = to_vvq(_vq); in virtqueue_disable_cb() local
1922 if (vq->packed_ring) in virtqueue_disable_cb()
1943 struct vring_virtqueue *vq = to_vvq(_vq); in virtqueue_enable_cb_prepare() local
1945 return vq->packed_ring ? virtqueue_enable_cb_prepare_packed(_vq) : in virtqueue_enable_cb_prepare()
1961 struct vring_virtqueue *vq = to_vvq(_vq); in virtqueue_poll() local
1963 virtio_mb(vq->weak_barriers); in virtqueue_poll()
1964 return vq->packed_ring ? virtqueue_poll_packed(_vq, last_used_idx) : in virtqueue_poll()
2003 struct vring_virtqueue *vq = to_vvq(_vq); in virtqueue_enable_cb_delayed() local
2005 return vq->packed_ring ? virtqueue_enable_cb_delayed_packed(_vq) : in virtqueue_enable_cb_delayed()
2020 struct vring_virtqueue *vq = to_vvq(_vq); in virtqueue_detach_unused_buf() local
2022 return vq->packed_ring ? virtqueue_detach_unused_buf_packed(_vq) : in virtqueue_detach_unused_buf()
2027 static inline bool more_used(const struct vring_virtqueue *vq) in more_used() argument
2029 return vq->packed_ring ? more_used_packed(vq) : more_used_split(vq); in more_used()
2034 struct vring_virtqueue *vq = to_vvq(_vq); in vring_interrupt() local
2036 if (!more_used(vq)) { in vring_interrupt()
2037 pr_debug("virtqueue interrupt with no work for %p\n", vq); in vring_interrupt()
2041 if (unlikely(vq->broken)) in vring_interrupt()
2044 pr_debug("virtqueue callback for %p (%p)\n", vq, vq->vq.callback); in vring_interrupt()
2045 if (vq->vq.callback) in vring_interrupt()
2046 vq->vq.callback(&vq->vq); in vring_interrupt()
2063 struct vring_virtqueue *vq; in __vring_new_virtqueue() local
2068 vq = kmalloc(sizeof(*vq), GFP_KERNEL); in __vring_new_virtqueue()
2069 if (!vq) in __vring_new_virtqueue()
2072 vq->packed_ring = false; in __vring_new_virtqueue()
2073 vq->vq.callback = callback; in __vring_new_virtqueue()
2074 vq->vq.vdev = vdev; in __vring_new_virtqueue()
2075 vq->vq.name = name; in __vring_new_virtqueue()
2076 vq->vq.num_free = vring.num; in __vring_new_virtqueue()
2077 vq->vq.index = index; in __vring_new_virtqueue()
2078 vq->we_own_ring = false; in __vring_new_virtqueue()
2079 vq->notify = notify; in __vring_new_virtqueue()
2080 vq->weak_barriers = weak_barriers; in __vring_new_virtqueue()
2081 vq->broken = false; in __vring_new_virtqueue()
2082 vq->last_used_idx = 0; in __vring_new_virtqueue()
2083 vq->num_added = 0; in __vring_new_virtqueue()
2084 vq->use_dma_api = vring_use_dma_api(vdev); in __vring_new_virtqueue()
2085 list_add_tail(&vq->vq.list, &vdev->vqs); in __vring_new_virtqueue()
2087 vq->in_use = false; in __vring_new_virtqueue()
2088 vq->last_add_time_valid = false; in __vring_new_virtqueue()
2091 vq->indirect = virtio_has_feature(vdev, VIRTIO_RING_F_INDIRECT_DESC) && in __vring_new_virtqueue()
2093 vq->event = virtio_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX); in __vring_new_virtqueue()
2096 vq->weak_barriers = false; in __vring_new_virtqueue()
2098 vq->split.queue_dma_addr = 0; in __vring_new_virtqueue()
2099 vq->split.queue_size_in_bytes = 0; in __vring_new_virtqueue()
2101 vq->split.vring = vring; in __vring_new_virtqueue()
2102 vq->split.avail_flags_shadow = 0; in __vring_new_virtqueue()
2103 vq->split.avail_idx_shadow = 0; in __vring_new_virtqueue()
2107 vq->split.avail_flags_shadow |= VRING_AVAIL_F_NO_INTERRUPT; in __vring_new_virtqueue()
2108 if (!vq->event) in __vring_new_virtqueue()
2109 vq->split.vring.avail->flags = cpu_to_virtio16(vdev, in __vring_new_virtqueue()
2110 vq->split.avail_flags_shadow); in __vring_new_virtqueue()
2113 vq->split.desc_state = kmalloc_array(vring.num, in __vring_new_virtqueue()
2115 if (!vq->split.desc_state) { in __vring_new_virtqueue()
2116 kfree(vq); in __vring_new_virtqueue()
2121 vq->free_head = 0; in __vring_new_virtqueue()
2123 vq->split.vring.desc[i].next = cpu_to_virtio16(vdev, i + 1); in __vring_new_virtqueue()
2124 memset(vq->split.desc_state, 0, vring.num * in __vring_new_virtqueue()
2127 return &vq->vq; in __vring_new_virtqueue()
2163 bool (*notify)(struct virtqueue *vq), in vring_new_virtqueue() argument
2164 void (*callback)(struct virtqueue *vq), in vring_new_virtqueue() argument
2180 struct vring_virtqueue *vq = to_vvq(_vq); in vring_del_virtqueue() local
2182 if (vq->we_own_ring) { in vring_del_virtqueue()
2183 if (vq->packed_ring) { in vring_del_virtqueue()
2184 vring_free_queue(vq->vq.vdev, in vring_del_virtqueue()
2185 vq->packed.ring_size_in_bytes, in vring_del_virtqueue()
2186 vq->packed.vring.desc, in vring_del_virtqueue()
2187 vq->packed.ring_dma_addr); in vring_del_virtqueue()
2189 vring_free_queue(vq->vq.vdev, in vring_del_virtqueue()
2190 vq->packed.event_size_in_bytes, in vring_del_virtqueue()
2191 vq->packed.vring.driver, in vring_del_virtqueue()
2192 vq->packed.driver_event_dma_addr); in vring_del_virtqueue()
2194 vring_free_queue(vq->vq.vdev, in vring_del_virtqueue()
2195 vq->packed.event_size_in_bytes, in vring_del_virtqueue()
2196 vq->packed.vring.device, in vring_del_virtqueue()
2197 vq->packed.device_event_dma_addr); in vring_del_virtqueue()
2199 kfree(vq->packed.desc_state); in vring_del_virtqueue()
2200 kfree(vq->packed.desc_extra); in vring_del_virtqueue()
2202 vring_free_queue(vq->vq.vdev, in vring_del_virtqueue()
2203 vq->split.queue_size_in_bytes, in vring_del_virtqueue()
2204 vq->split.vring.desc, in vring_del_virtqueue()
2205 vq->split.queue_dma_addr); in vring_del_virtqueue()
2207 kfree(vq->split.desc_state); in vring_del_virtqueue()
2211 kfree(vq); in vring_del_virtqueue()
2252 struct vring_virtqueue *vq = to_vvq(_vq); in virtqueue_get_vring_size() local
2254 return vq->packed_ring ? vq->packed.vring.num : vq->split.vring.num; in virtqueue_get_vring_size()
2260 struct vring_virtqueue *vq = to_vvq(_vq); in virtqueue_is_broken() local
2262 return vq->broken; in virtqueue_is_broken()
2275 struct vring_virtqueue *vq = to_vvq(_vq); in virtio_break_device() local
2276 vq->broken = true; in virtio_break_device()
2283 struct vring_virtqueue *vq = to_vvq(_vq); in virtqueue_get_desc_addr() local
2285 BUG_ON(!vq->we_own_ring); in virtqueue_get_desc_addr()
2287 if (vq->packed_ring) in virtqueue_get_desc_addr()
2288 return vq->packed.ring_dma_addr; in virtqueue_get_desc_addr()
2290 return vq->split.queue_dma_addr; in virtqueue_get_desc_addr()
2296 struct vring_virtqueue *vq = to_vvq(_vq); in virtqueue_get_avail_addr() local
2298 BUG_ON(!vq->we_own_ring); in virtqueue_get_avail_addr()
2300 if (vq->packed_ring) in virtqueue_get_avail_addr()
2301 return vq->packed.driver_event_dma_addr; in virtqueue_get_avail_addr()
2303 return vq->split.queue_dma_addr + in virtqueue_get_avail_addr()
2304 ((char *)vq->split.vring.avail - (char *)vq->split.vring.desc); in virtqueue_get_avail_addr()
2310 struct vring_virtqueue *vq = to_vvq(_vq); in virtqueue_get_used_addr() local
2312 BUG_ON(!vq->we_own_ring); in virtqueue_get_used_addr()
2314 if (vq->packed_ring) in virtqueue_get_used_addr()
2315 return vq->packed.device_event_dma_addr; in virtqueue_get_used_addr()
2317 return vq->split.queue_dma_addr + in virtqueue_get_used_addr()
2318 ((char *)vq->split.vring.used - (char *)vq->split.vring.desc); in virtqueue_get_used_addr()
2323 const struct vring *virtqueue_get_vring(struct virtqueue *vq) in virtqueue_get_vring() argument
2325 return &to_vvq(vq)->split.vring; in virtqueue_get_vring()