Lines Matching refs:vq
22 dev_err(&(_vq)->vq.vdev->dev, \
23 "%s:"fmt, (_vq)->vq.name, ##args); \
31 (_vq)->vq.name, (_vq)->in_use); \
59 dev_err(&_vq->vq.vdev->dev, \
60 "%s:"fmt, (_vq)->vq.name, ##args); \
63 #define START_USE(vq) argument
64 #define END_USE(vq) argument
65 #define LAST_ADD_TIME_UPDATE(vq) argument
66 #define LAST_ADD_TIME_CHECK(vq) argument
67 #define LAST_ADD_TIME_INVALID(vq) argument
155 struct virtqueue vq; member
200 bool (*notify)(struct virtqueue *vq);
230 #define to_vvq(_vq) container_of(_vq, struct vring_virtqueue, vq)
232 static inline bool virtqueue_use_indirect(struct vring_virtqueue *vq, in virtqueue_use_indirect() argument
239 return (vq->indirect && total_sg > 1 && vq->vq.num_free); in virtqueue_use_indirect()
346 static inline struct device *vring_dma_dev(const struct vring_virtqueue *vq) in vring_dma_dev() argument
348 return vq->vq.vdev->dev.parent; in vring_dma_dev()
352 static dma_addr_t vring_map_one_sg(const struct vring_virtqueue *vq, in vring_map_one_sg() argument
356 if (!vq->use_dma_api) { in vring_map_one_sg()
371 return dma_map_page(vring_dma_dev(vq), in vring_map_one_sg()
376 static dma_addr_t vring_map_single(const struct vring_virtqueue *vq, in vring_map_single() argument
380 if (!vq->use_dma_api) in vring_map_single()
383 return dma_map_single(vring_dma_dev(vq), in vring_map_single()
387 static int vring_mapping_error(const struct vring_virtqueue *vq, in vring_mapping_error() argument
390 if (!vq->use_dma_api) in vring_mapping_error()
393 return dma_mapping_error(vring_dma_dev(vq), addr); in vring_mapping_error()
396 static void virtqueue_init(struct vring_virtqueue *vq, u32 num) in virtqueue_init() argument
398 vq->vq.num_free = num; in virtqueue_init()
400 if (vq->packed_ring) in virtqueue_init()
401 vq->last_used_idx = 0 | (1 << VRING_PACKED_EVENT_F_WRAP_CTR); in virtqueue_init()
403 vq->last_used_idx = 0; in virtqueue_init()
405 vq->event_triggered = false; in virtqueue_init()
406 vq->num_added = 0; in virtqueue_init()
409 vq->in_use = false; in virtqueue_init()
410 vq->last_add_time_valid = false; in virtqueue_init()
419 static void vring_unmap_one_split_indirect(const struct vring_virtqueue *vq, in vring_unmap_one_split_indirect() argument
424 if (!vq->use_dma_api) in vring_unmap_one_split_indirect()
427 flags = virtio16_to_cpu(vq->vq.vdev, desc->flags); in vring_unmap_one_split_indirect()
429 dma_unmap_page(vring_dma_dev(vq), in vring_unmap_one_split_indirect()
430 virtio64_to_cpu(vq->vq.vdev, desc->addr), in vring_unmap_one_split_indirect()
431 virtio32_to_cpu(vq->vq.vdev, desc->len), in vring_unmap_one_split_indirect()
436 static unsigned int vring_unmap_one_split(const struct vring_virtqueue *vq, in vring_unmap_one_split() argument
439 struct vring_desc_extra *extra = vq->split.desc_extra; in vring_unmap_one_split()
442 if (!vq->use_dma_api) in vring_unmap_one_split()
448 dma_unmap_single(vring_dma_dev(vq), in vring_unmap_one_split()
454 dma_unmap_page(vring_dma_dev(vq), in vring_unmap_one_split()
488 static inline unsigned int virtqueue_add_desc_split(struct virtqueue *vq, in virtqueue_add_desc_split() argument
496 struct vring_virtqueue *vring = to_vvq(vq); in virtqueue_add_desc_split()
500 desc[i].flags = cpu_to_virtio16(vq->vdev, flags); in virtqueue_add_desc_split()
501 desc[i].addr = cpu_to_virtio64(vq->vdev, addr); in virtqueue_add_desc_split()
502 desc[i].len = cpu_to_virtio32(vq->vdev, len); in virtqueue_add_desc_split()
506 desc[i].next = cpu_to_virtio16(vq->vdev, next); in virtqueue_add_desc_split()
512 next = virtio16_to_cpu(vq->vdev, desc[i].next); in virtqueue_add_desc_split()
526 struct vring_virtqueue *vq = to_vvq(_vq); in virtqueue_add_split() local
533 START_USE(vq); in virtqueue_add_split()
536 BUG_ON(ctx && vq->indirect); in virtqueue_add_split()
538 if (unlikely(vq->broken)) { in virtqueue_add_split()
539 END_USE(vq); in virtqueue_add_split()
543 LAST_ADD_TIME_UPDATE(vq); in virtqueue_add_split()
547 head = vq->free_head; in virtqueue_add_split()
549 if (virtqueue_use_indirect(vq, total_sg)) in virtqueue_add_split()
553 WARN_ON_ONCE(total_sg > vq->split.vring.num && !vq->indirect); in virtqueue_add_split()
564 desc = vq->split.vring.desc; in virtqueue_add_split()
569 if (unlikely(vq->vq.num_free < descs_used)) { in virtqueue_add_split()
571 descs_used, vq->vq.num_free); in virtqueue_add_split()
576 vq->notify(&vq->vq); in virtqueue_add_split()
579 END_USE(vq); in virtqueue_add_split()
585 dma_addr_t addr = vring_map_one_sg(vq, sg, DMA_TO_DEVICE); in virtqueue_add_split()
586 if (vring_mapping_error(vq, addr)) in virtqueue_add_split()
600 dma_addr_t addr = vring_map_one_sg(vq, sg, DMA_FROM_DEVICE); in virtqueue_add_split()
601 if (vring_mapping_error(vq, addr)) in virtqueue_add_split()
617 if (!indirect && vq->use_dma_api) in virtqueue_add_split()
618 vq->split.desc_extra[prev & (vq->split.vring.num - 1)].flags &= in virtqueue_add_split()
624 vq, desc, total_sg * sizeof(struct vring_desc), in virtqueue_add_split()
626 if (vring_mapping_error(vq, addr)) in virtqueue_add_split()
629 virtqueue_add_desc_split(_vq, vq->split.vring.desc, in virtqueue_add_split()
637 vq->vq.num_free -= descs_used; in virtqueue_add_split()
641 vq->free_head = vq->split.desc_extra[head].next; in virtqueue_add_split()
643 vq->free_head = i; in virtqueue_add_split()
646 vq->split.desc_state[head].data = data; in virtqueue_add_split()
648 vq->split.desc_state[head].indir_desc = desc; in virtqueue_add_split()
650 vq->split.desc_state[head].indir_desc = ctx; in virtqueue_add_split()
654 avail = vq->split.avail_idx_shadow & (vq->split.vring.num - 1); in virtqueue_add_split()
655 vq->split.vring.avail->ring[avail] = cpu_to_virtio16(_vq->vdev, head); in virtqueue_add_split()
659 virtio_wmb(vq->weak_barriers); in virtqueue_add_split()
660 vq->split.avail_idx_shadow++; in virtqueue_add_split()
661 vq->split.vring.avail->idx = cpu_to_virtio16(_vq->vdev, in virtqueue_add_split()
662 vq->split.avail_idx_shadow); in virtqueue_add_split()
663 vq->num_added++; in virtqueue_add_split()
665 pr_debug("Added buffer head %i to %p\n", head, vq); in virtqueue_add_split()
666 END_USE(vq); in virtqueue_add_split()
670 if (unlikely(vq->num_added == (1 << 16) - 1)) in virtqueue_add_split()
687 vring_unmap_one_split_indirect(vq, &desc[i]); in virtqueue_add_split()
690 i = vring_unmap_one_split(vq, i); in virtqueue_add_split()
696 END_USE(vq); in virtqueue_add_split()
702 struct vring_virtqueue *vq = to_vvq(_vq); in virtqueue_kick_prepare_split() local
706 START_USE(vq); in virtqueue_kick_prepare_split()
709 virtio_mb(vq->weak_barriers); in virtqueue_kick_prepare_split()
711 old = vq->split.avail_idx_shadow - vq->num_added; in virtqueue_kick_prepare_split()
712 new = vq->split.avail_idx_shadow; in virtqueue_kick_prepare_split()
713 vq->num_added = 0; in virtqueue_kick_prepare_split()
715 LAST_ADD_TIME_CHECK(vq); in virtqueue_kick_prepare_split()
716 LAST_ADD_TIME_INVALID(vq); in virtqueue_kick_prepare_split()
718 if (vq->event) { in virtqueue_kick_prepare_split()
720 vring_avail_event(&vq->split.vring)), in virtqueue_kick_prepare_split()
723 needs_kick = !(vq->split.vring.used->flags & in virtqueue_kick_prepare_split()
727 END_USE(vq); in virtqueue_kick_prepare_split()
731 static void detach_buf_split(struct vring_virtqueue *vq, unsigned int head, in detach_buf_split() argument
735 __virtio16 nextflag = cpu_to_virtio16(vq->vq.vdev, VRING_DESC_F_NEXT); in detach_buf_split()
738 vq->split.desc_state[head].data = NULL; in detach_buf_split()
743 while (vq->split.vring.desc[i].flags & nextflag) { in detach_buf_split()
744 vring_unmap_one_split(vq, i); in detach_buf_split()
745 i = vq->split.desc_extra[i].next; in detach_buf_split()
746 vq->vq.num_free++; in detach_buf_split()
749 vring_unmap_one_split(vq, i); in detach_buf_split()
750 vq->split.desc_extra[i].next = vq->free_head; in detach_buf_split()
751 vq->free_head = head; in detach_buf_split()
754 vq->vq.num_free++; in detach_buf_split()
756 if (vq->indirect) { in detach_buf_split()
758 vq->split.desc_state[head].indir_desc; in detach_buf_split()
765 len = vq->split.desc_extra[head].len; in detach_buf_split()
767 BUG_ON(!(vq->split.desc_extra[head].flags & in detach_buf_split()
772 vring_unmap_one_split_indirect(vq, &indir_desc[j]); in detach_buf_split()
775 vq->split.desc_state[head].indir_desc = NULL; in detach_buf_split()
777 *ctx = vq->split.desc_state[head].indir_desc; in detach_buf_split()
781 static inline bool more_used_split(const struct vring_virtqueue *vq) in more_used_split() argument
783 return vq->last_used_idx != virtio16_to_cpu(vq->vq.vdev, in more_used_split()
784 vq->split.vring.used->idx); in more_used_split()
791 struct vring_virtqueue *vq = to_vvq(_vq); in virtqueue_get_buf_ctx_split() local
796 START_USE(vq); in virtqueue_get_buf_ctx_split()
798 if (unlikely(vq->broken)) { in virtqueue_get_buf_ctx_split()
799 END_USE(vq); in virtqueue_get_buf_ctx_split()
803 if (!more_used_split(vq)) { in virtqueue_get_buf_ctx_split()
805 END_USE(vq); in virtqueue_get_buf_ctx_split()
810 virtio_rmb(vq->weak_barriers); in virtqueue_get_buf_ctx_split()
812 last_used = (vq->last_used_idx & (vq->split.vring.num - 1)); in virtqueue_get_buf_ctx_split()
814 vq->split.vring.used->ring[last_used].id); in virtqueue_get_buf_ctx_split()
816 vq->split.vring.used->ring[last_used].len); in virtqueue_get_buf_ctx_split()
818 if (unlikely(i >= vq->split.vring.num)) { in virtqueue_get_buf_ctx_split()
819 BAD_RING(vq, "id %u out of range\n", i); in virtqueue_get_buf_ctx_split()
822 if (unlikely(!vq->split.desc_state[i].data)) { in virtqueue_get_buf_ctx_split()
823 BAD_RING(vq, "id %u is not a head!\n", i); in virtqueue_get_buf_ctx_split()
828 ret = vq->split.desc_state[i].data; in virtqueue_get_buf_ctx_split()
829 detach_buf_split(vq, i, ctx); in virtqueue_get_buf_ctx_split()
830 vq->last_used_idx++; in virtqueue_get_buf_ctx_split()
834 if (!(vq->split.avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT)) in virtqueue_get_buf_ctx_split()
835 virtio_store_mb(vq->weak_barriers, in virtqueue_get_buf_ctx_split()
836 &vring_used_event(&vq->split.vring), in virtqueue_get_buf_ctx_split()
837 cpu_to_virtio16(_vq->vdev, vq->last_used_idx)); in virtqueue_get_buf_ctx_split()
839 LAST_ADD_TIME_INVALID(vq); in virtqueue_get_buf_ctx_split()
841 END_USE(vq); in virtqueue_get_buf_ctx_split()
847 struct vring_virtqueue *vq = to_vvq(_vq); in virtqueue_disable_cb_split() local
849 if (!(vq->split.avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT)) { in virtqueue_disable_cb_split()
850 vq->split.avail_flags_shadow |= VRING_AVAIL_F_NO_INTERRUPT; in virtqueue_disable_cb_split()
851 if (vq->event) in virtqueue_disable_cb_split()
853 vring_used_event(&vq->split.vring) = 0x0; in virtqueue_disable_cb_split()
855 vq->split.vring.avail->flags = in virtqueue_disable_cb_split()
857 vq->split.avail_flags_shadow); in virtqueue_disable_cb_split()
863 struct vring_virtqueue *vq = to_vvq(_vq); in virtqueue_enable_cb_prepare_split() local
866 START_USE(vq); in virtqueue_enable_cb_prepare_split()
873 if (vq->split.avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT) { in virtqueue_enable_cb_prepare_split()
874 vq->split.avail_flags_shadow &= ~VRING_AVAIL_F_NO_INTERRUPT; in virtqueue_enable_cb_prepare_split()
875 if (!vq->event) in virtqueue_enable_cb_prepare_split()
876 vq->split.vring.avail->flags = in virtqueue_enable_cb_prepare_split()
878 vq->split.avail_flags_shadow); in virtqueue_enable_cb_prepare_split()
880 vring_used_event(&vq->split.vring) = cpu_to_virtio16(_vq->vdev, in virtqueue_enable_cb_prepare_split()
881 last_used_idx = vq->last_used_idx); in virtqueue_enable_cb_prepare_split()
882 END_USE(vq); in virtqueue_enable_cb_prepare_split()
888 struct vring_virtqueue *vq = to_vvq(_vq); in virtqueue_poll_split() local
891 vq->split.vring.used->idx); in virtqueue_poll_split()
896 struct vring_virtqueue *vq = to_vvq(_vq); in virtqueue_enable_cb_delayed_split() local
899 START_USE(vq); in virtqueue_enable_cb_delayed_split()
906 if (vq->split.avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT) { in virtqueue_enable_cb_delayed_split()
907 vq->split.avail_flags_shadow &= ~VRING_AVAIL_F_NO_INTERRUPT; in virtqueue_enable_cb_delayed_split()
908 if (!vq->event) in virtqueue_enable_cb_delayed_split()
909 vq->split.vring.avail->flags = in virtqueue_enable_cb_delayed_split()
911 vq->split.avail_flags_shadow); in virtqueue_enable_cb_delayed_split()
914 bufs = (u16)(vq->split.avail_idx_shadow - vq->last_used_idx) * 3 / 4; in virtqueue_enable_cb_delayed_split()
916 virtio_store_mb(vq->weak_barriers, in virtqueue_enable_cb_delayed_split()
917 &vring_used_event(&vq->split.vring), in virtqueue_enable_cb_delayed_split()
918 cpu_to_virtio16(_vq->vdev, vq->last_used_idx + bufs)); in virtqueue_enable_cb_delayed_split()
920 if (unlikely((u16)(virtio16_to_cpu(_vq->vdev, vq->split.vring.used->idx) in virtqueue_enable_cb_delayed_split()
921 - vq->last_used_idx) > bufs)) { in virtqueue_enable_cb_delayed_split()
922 END_USE(vq); in virtqueue_enable_cb_delayed_split()
926 END_USE(vq); in virtqueue_enable_cb_delayed_split()
932 struct vring_virtqueue *vq = to_vvq(_vq); in virtqueue_detach_unused_buf_split() local
936 START_USE(vq); in virtqueue_detach_unused_buf_split()
938 for (i = 0; i < vq->split.vring.num; i++) { in virtqueue_detach_unused_buf_split()
939 if (!vq->split.desc_state[i].data) in virtqueue_detach_unused_buf_split()
942 buf = vq->split.desc_state[i].data; in virtqueue_detach_unused_buf_split()
943 detach_buf_split(vq, i, NULL); in virtqueue_detach_unused_buf_split()
944 vq->split.avail_idx_shadow--; in virtqueue_detach_unused_buf_split()
945 vq->split.vring.avail->idx = cpu_to_virtio16(_vq->vdev, in virtqueue_detach_unused_buf_split()
946 vq->split.avail_idx_shadow); in virtqueue_detach_unused_buf_split()
947 END_USE(vq); in virtqueue_detach_unused_buf_split()
951 BUG_ON(vq->vq.num_free != vq->split.vring.num); in virtqueue_detach_unused_buf_split()
953 END_USE(vq); in virtqueue_detach_unused_buf_split()
958 struct vring_virtqueue *vq) in virtqueue_vring_init_split() argument
962 vdev = vq->vq.vdev; in virtqueue_vring_init_split()
968 if (!vq->vq.callback) { in virtqueue_vring_init_split()
970 if (!vq->event) in virtqueue_vring_init_split()
976 static void virtqueue_reinit_split(struct vring_virtqueue *vq) in virtqueue_reinit_split() argument
980 num = vq->split.vring.num; in virtqueue_reinit_split()
982 vq->split.vring.avail->flags = 0; in virtqueue_reinit_split()
983 vq->split.vring.avail->idx = 0; in virtqueue_reinit_split()
986 vq->split.vring.avail->ring[num] = 0; in virtqueue_reinit_split()
988 vq->split.vring.used->flags = 0; in virtqueue_reinit_split()
989 vq->split.vring.used->idx = 0; in virtqueue_reinit_split()
992 *(__virtio16 *)&(vq->split.vring.used->ring[num]) = 0; in virtqueue_reinit_split()
994 virtqueue_init(vq, num); in virtqueue_reinit_split()
996 virtqueue_vring_init_split(&vq->split, vq); in virtqueue_reinit_split()
999 static void virtqueue_vring_attach_split(struct vring_virtqueue *vq, in virtqueue_vring_attach_split() argument
1002 vq->split = *vring_split; in virtqueue_vring_attach_split()
1005 vq->free_head = 0; in virtqueue_vring_attach_split()
1106 struct virtqueue *vq; in vring_create_virtqueue_split() local
1114 vq = __vring_new_virtqueue(index, &vring_split, vdev, weak_barriers, in vring_create_virtqueue_split()
1116 if (!vq) { in vring_create_virtqueue_split()
1121 to_vvq(vq)->we_own_ring = true; in vring_create_virtqueue_split()
1123 return vq; in vring_create_virtqueue_split()
1129 struct vring_virtqueue *vq = to_vvq(_vq); in virtqueue_resize_split() local
1134 vq->split.vring_align, in virtqueue_resize_split()
1135 vq->split.may_reduce_num); in virtqueue_resize_split()
1143 vring_free(&vq->vq); in virtqueue_resize_split()
1145 virtqueue_vring_init_split(&vring_split, vq); in virtqueue_resize_split()
1147 virtqueue_init(vq, vring_split.vring.num); in virtqueue_resize_split()
1148 virtqueue_vring_attach_split(vq, &vring_split); in virtqueue_resize_split()
1155 virtqueue_reinit_split(vq); in virtqueue_resize_split()
1173 static void vring_unmap_extra_packed(const struct vring_virtqueue *vq, in vring_unmap_extra_packed() argument
1178 if (!vq->use_dma_api) in vring_unmap_extra_packed()
1184 dma_unmap_single(vring_dma_dev(vq), in vring_unmap_extra_packed()
1189 dma_unmap_page(vring_dma_dev(vq), in vring_unmap_extra_packed()
1196 static void vring_unmap_desc_packed(const struct vring_virtqueue *vq, in vring_unmap_desc_packed() argument
1201 if (!vq->use_dma_api) in vring_unmap_desc_packed()
1206 dma_unmap_page(vring_dma_dev(vq), in vring_unmap_desc_packed()
1230 static int virtqueue_add_indirect_packed(struct vring_virtqueue *vq, in virtqueue_add_indirect_packed() argument
1244 head = vq->packed.next_avail_idx; in virtqueue_add_indirect_packed()
1249 if (unlikely(vq->vq.num_free < 1)) { in virtqueue_add_indirect_packed()
1252 END_USE(vq); in virtqueue_add_indirect_packed()
1257 id = vq->free_head; in virtqueue_add_indirect_packed()
1258 BUG_ON(id == vq->packed.vring.num); in virtqueue_add_indirect_packed()
1262 addr = vring_map_one_sg(vq, sg, n < out_sgs ? in virtqueue_add_indirect_packed()
1264 if (vring_mapping_error(vq, addr)) in virtqueue_add_indirect_packed()
1276 addr = vring_map_single(vq, desc, in virtqueue_add_indirect_packed()
1279 if (vring_mapping_error(vq, addr)) in virtqueue_add_indirect_packed()
1282 vq->packed.vring.desc[head].addr = cpu_to_le64(addr); in virtqueue_add_indirect_packed()
1283 vq->packed.vring.desc[head].len = cpu_to_le32(total_sg * in virtqueue_add_indirect_packed()
1285 vq->packed.vring.desc[head].id = cpu_to_le16(id); in virtqueue_add_indirect_packed()
1287 if (vq->use_dma_api) { in virtqueue_add_indirect_packed()
1288 vq->packed.desc_extra[id].addr = addr; in virtqueue_add_indirect_packed()
1289 vq->packed.desc_extra[id].len = total_sg * in virtqueue_add_indirect_packed()
1291 vq->packed.desc_extra[id].flags = VRING_DESC_F_INDIRECT | in virtqueue_add_indirect_packed()
1292 vq->packed.avail_used_flags; in virtqueue_add_indirect_packed()
1300 virtio_wmb(vq->weak_barriers); in virtqueue_add_indirect_packed()
1301 vq->packed.vring.desc[head].flags = cpu_to_le16(VRING_DESC_F_INDIRECT | in virtqueue_add_indirect_packed()
1302 vq->packed.avail_used_flags); in virtqueue_add_indirect_packed()
1305 vq->vq.num_free -= 1; in virtqueue_add_indirect_packed()
1309 if (n >= vq->packed.vring.num) { in virtqueue_add_indirect_packed()
1311 vq->packed.avail_wrap_counter ^= 1; in virtqueue_add_indirect_packed()
1312 vq->packed.avail_used_flags ^= in virtqueue_add_indirect_packed()
1316 vq->packed.next_avail_idx = n; in virtqueue_add_indirect_packed()
1317 vq->free_head = vq->packed.desc_extra[id].next; in virtqueue_add_indirect_packed()
1320 vq->packed.desc_state[id].num = 1; in virtqueue_add_indirect_packed()
1321 vq->packed.desc_state[id].data = data; in virtqueue_add_indirect_packed()
1322 vq->packed.desc_state[id].indir_desc = desc; in virtqueue_add_indirect_packed()
1323 vq->packed.desc_state[id].last = id; in virtqueue_add_indirect_packed()
1325 vq->num_added += 1; in virtqueue_add_indirect_packed()
1327 pr_debug("Added buffer head %i to %p\n", head, vq); in virtqueue_add_indirect_packed()
1328 END_USE(vq); in virtqueue_add_indirect_packed()
1336 vring_unmap_desc_packed(vq, &desc[i]); in virtqueue_add_indirect_packed()
1340 END_USE(vq); in virtqueue_add_indirect_packed()
1353 struct vring_virtqueue *vq = to_vvq(_vq); in virtqueue_add_packed() local
1361 START_USE(vq); in virtqueue_add_packed()
1364 BUG_ON(ctx && vq->indirect); in virtqueue_add_packed()
1366 if (unlikely(vq->broken)) { in virtqueue_add_packed()
1367 END_USE(vq); in virtqueue_add_packed()
1371 LAST_ADD_TIME_UPDATE(vq); in virtqueue_add_packed()
1375 if (virtqueue_use_indirect(vq, total_sg)) { in virtqueue_add_packed()
1376 err = virtqueue_add_indirect_packed(vq, sgs, total_sg, out_sgs, in virtqueue_add_packed()
1379 END_USE(vq); in virtqueue_add_packed()
1386 head = vq->packed.next_avail_idx; in virtqueue_add_packed()
1387 avail_used_flags = vq->packed.avail_used_flags; in virtqueue_add_packed()
1389 WARN_ON_ONCE(total_sg > vq->packed.vring.num && !vq->indirect); in virtqueue_add_packed()
1391 desc = vq->packed.vring.desc; in virtqueue_add_packed()
1395 if (unlikely(vq->vq.num_free < descs_used)) { in virtqueue_add_packed()
1397 descs_used, vq->vq.num_free); in virtqueue_add_packed()
1398 END_USE(vq); in virtqueue_add_packed()
1402 id = vq->free_head; in virtqueue_add_packed()
1403 BUG_ON(id == vq->packed.vring.num); in virtqueue_add_packed()
1409 dma_addr_t addr = vring_map_one_sg(vq, sg, n < out_sgs ? in virtqueue_add_packed()
1411 if (vring_mapping_error(vq, addr)) in virtqueue_add_packed()
1414 flags = cpu_to_le16(vq->packed.avail_used_flags | in virtqueue_add_packed()
1426 if (unlikely(vq->use_dma_api)) { in virtqueue_add_packed()
1427 vq->packed.desc_extra[curr].addr = addr; in virtqueue_add_packed()
1428 vq->packed.desc_extra[curr].len = sg->length; in virtqueue_add_packed()
1429 vq->packed.desc_extra[curr].flags = in virtqueue_add_packed()
1433 curr = vq->packed.desc_extra[curr].next; in virtqueue_add_packed()
1435 if ((unlikely(++i >= vq->packed.vring.num))) { in virtqueue_add_packed()
1437 vq->packed.avail_used_flags ^= in virtqueue_add_packed()
1445 vq->packed.avail_wrap_counter ^= 1; in virtqueue_add_packed()
1448 vq->vq.num_free -= descs_used; in virtqueue_add_packed()
1451 vq->packed.next_avail_idx = i; in virtqueue_add_packed()
1452 vq->free_head = curr; in virtqueue_add_packed()
1455 vq->packed.desc_state[id].num = descs_used; in virtqueue_add_packed()
1456 vq->packed.desc_state[id].data = data; in virtqueue_add_packed()
1457 vq->packed.desc_state[id].indir_desc = ctx; in virtqueue_add_packed()
1458 vq->packed.desc_state[id].last = prev; in virtqueue_add_packed()
1465 virtio_wmb(vq->weak_barriers); in virtqueue_add_packed()
1466 vq->packed.vring.desc[head].flags = head_flags; in virtqueue_add_packed()
1467 vq->num_added += descs_used; in virtqueue_add_packed()
1469 pr_debug("Added buffer head %i to %p\n", head, vq); in virtqueue_add_packed()
1470 END_USE(vq); in virtqueue_add_packed()
1477 curr = vq->free_head; in virtqueue_add_packed()
1479 vq->packed.avail_used_flags = avail_used_flags; in virtqueue_add_packed()
1484 vring_unmap_extra_packed(vq, &vq->packed.desc_extra[curr]); in virtqueue_add_packed()
1485 curr = vq->packed.desc_extra[curr].next; in virtqueue_add_packed()
1487 if (i >= vq->packed.vring.num) in virtqueue_add_packed()
1491 END_USE(vq); in virtqueue_add_packed()
1497 struct vring_virtqueue *vq = to_vvq(_vq); in virtqueue_kick_prepare_packed() local
1508 START_USE(vq); in virtqueue_kick_prepare_packed()
1514 virtio_mb(vq->weak_barriers); in virtqueue_kick_prepare_packed()
1516 old = vq->packed.next_avail_idx - vq->num_added; in virtqueue_kick_prepare_packed()
1517 new = vq->packed.next_avail_idx; in virtqueue_kick_prepare_packed()
1518 vq->num_added = 0; in virtqueue_kick_prepare_packed()
1520 snapshot.u32 = *(u32 *)vq->packed.vring.device; in virtqueue_kick_prepare_packed()
1523 LAST_ADD_TIME_CHECK(vq); in virtqueue_kick_prepare_packed()
1524 LAST_ADD_TIME_INVALID(vq); in virtqueue_kick_prepare_packed()
1535 if (wrap_counter != vq->packed.avail_wrap_counter) in virtqueue_kick_prepare_packed()
1536 event_idx -= vq->packed.vring.num; in virtqueue_kick_prepare_packed()
1540 END_USE(vq); in virtqueue_kick_prepare_packed()
1544 static void detach_buf_packed(struct vring_virtqueue *vq, in detach_buf_packed() argument
1551 state = &vq->packed.desc_state[id]; in detach_buf_packed()
1556 vq->packed.desc_extra[state->last].next = vq->free_head; in detach_buf_packed()
1557 vq->free_head = id; in detach_buf_packed()
1558 vq->vq.num_free += state->num; in detach_buf_packed()
1560 if (unlikely(vq->use_dma_api)) { in detach_buf_packed()
1563 vring_unmap_extra_packed(vq, in detach_buf_packed()
1564 &vq->packed.desc_extra[curr]); in detach_buf_packed()
1565 curr = vq->packed.desc_extra[curr].next; in detach_buf_packed()
1569 if (vq->indirect) { in detach_buf_packed()
1577 if (vq->use_dma_api) { in detach_buf_packed()
1578 len = vq->packed.desc_extra[id].len; in detach_buf_packed()
1581 vring_unmap_desc_packed(vq, &desc[i]); in detach_buf_packed()
1590 static inline bool is_used_desc_packed(const struct vring_virtqueue *vq, in is_used_desc_packed() argument
1596 flags = le16_to_cpu(vq->packed.vring.desc[idx].flags); in is_used_desc_packed()
1603 static inline bool more_used_packed(const struct vring_virtqueue *vq) in more_used_packed() argument
1609 last_used_idx = READ_ONCE(vq->last_used_idx); in more_used_packed()
1612 return is_used_desc_packed(vq, last_used, used_wrap_counter); in more_used_packed()
1619 struct vring_virtqueue *vq = to_vvq(_vq); in virtqueue_get_buf_ctx_packed() local
1624 START_USE(vq); in virtqueue_get_buf_ctx_packed()
1626 if (unlikely(vq->broken)) { in virtqueue_get_buf_ctx_packed()
1627 END_USE(vq); in virtqueue_get_buf_ctx_packed()
1631 if (!more_used_packed(vq)) { in virtqueue_get_buf_ctx_packed()
1633 END_USE(vq); in virtqueue_get_buf_ctx_packed()
1638 virtio_rmb(vq->weak_barriers); in virtqueue_get_buf_ctx_packed()
1640 last_used_idx = READ_ONCE(vq->last_used_idx); in virtqueue_get_buf_ctx_packed()
1643 id = le16_to_cpu(vq->packed.vring.desc[last_used].id); in virtqueue_get_buf_ctx_packed()
1644 *len = le32_to_cpu(vq->packed.vring.desc[last_used].len); in virtqueue_get_buf_ctx_packed()
1646 if (unlikely(id >= vq->packed.vring.num)) { in virtqueue_get_buf_ctx_packed()
1647 BAD_RING(vq, "id %u out of range\n", id); in virtqueue_get_buf_ctx_packed()
1650 if (unlikely(!vq->packed.desc_state[id].data)) { in virtqueue_get_buf_ctx_packed()
1651 BAD_RING(vq, "id %u is not a head!\n", id); in virtqueue_get_buf_ctx_packed()
1656 ret = vq->packed.desc_state[id].data; in virtqueue_get_buf_ctx_packed()
1657 detach_buf_packed(vq, id, ctx); in virtqueue_get_buf_ctx_packed()
1659 last_used += vq->packed.desc_state[id].num; in virtqueue_get_buf_ctx_packed()
1660 if (unlikely(last_used >= vq->packed.vring.num)) { in virtqueue_get_buf_ctx_packed()
1661 last_used -= vq->packed.vring.num; in virtqueue_get_buf_ctx_packed()
1666 WRITE_ONCE(vq->last_used_idx, last_used); in virtqueue_get_buf_ctx_packed()
1673 if (vq->packed.event_flags_shadow == VRING_PACKED_EVENT_FLAG_DESC) in virtqueue_get_buf_ctx_packed()
1674 virtio_store_mb(vq->weak_barriers, in virtqueue_get_buf_ctx_packed()
1675 &vq->packed.vring.driver->off_wrap, in virtqueue_get_buf_ctx_packed()
1676 cpu_to_le16(vq->last_used_idx)); in virtqueue_get_buf_ctx_packed()
1678 LAST_ADD_TIME_INVALID(vq); in virtqueue_get_buf_ctx_packed()
1680 END_USE(vq); in virtqueue_get_buf_ctx_packed()
1686 struct vring_virtqueue *vq = to_vvq(_vq); in virtqueue_disable_cb_packed() local
1688 if (vq->packed.event_flags_shadow != VRING_PACKED_EVENT_FLAG_DISABLE) { in virtqueue_disable_cb_packed()
1689 vq->packed.event_flags_shadow = VRING_PACKED_EVENT_FLAG_DISABLE; in virtqueue_disable_cb_packed()
1690 vq->packed.vring.driver->flags = in virtqueue_disable_cb_packed()
1691 cpu_to_le16(vq->packed.event_flags_shadow); in virtqueue_disable_cb_packed()
1697 struct vring_virtqueue *vq = to_vvq(_vq); in virtqueue_enable_cb_prepare_packed() local
1699 START_USE(vq); in virtqueue_enable_cb_prepare_packed()
1706 if (vq->event) { in virtqueue_enable_cb_prepare_packed()
1707 vq->packed.vring.driver->off_wrap = in virtqueue_enable_cb_prepare_packed()
1708 cpu_to_le16(vq->last_used_idx); in virtqueue_enable_cb_prepare_packed()
1713 virtio_wmb(vq->weak_barriers); in virtqueue_enable_cb_prepare_packed()
1716 if (vq->packed.event_flags_shadow == VRING_PACKED_EVENT_FLAG_DISABLE) { in virtqueue_enable_cb_prepare_packed()
1717 vq->packed.event_flags_shadow = vq->event ? in virtqueue_enable_cb_prepare_packed()
1720 vq->packed.vring.driver->flags = in virtqueue_enable_cb_prepare_packed()
1721 cpu_to_le16(vq->packed.event_flags_shadow); in virtqueue_enable_cb_prepare_packed()
1724 END_USE(vq); in virtqueue_enable_cb_prepare_packed()
1725 return vq->last_used_idx; in virtqueue_enable_cb_prepare_packed()
1730 struct vring_virtqueue *vq = to_vvq(_vq); in virtqueue_poll_packed() local
1737 return is_used_desc_packed(vq, used_idx, wrap_counter); in virtqueue_poll_packed()
1742 struct vring_virtqueue *vq = to_vvq(_vq); in virtqueue_enable_cb_delayed_packed() local
1746 START_USE(vq); in virtqueue_enable_cb_delayed_packed()
1753 if (vq->event) { in virtqueue_enable_cb_delayed_packed()
1755 bufs = (vq->packed.vring.num - vq->vq.num_free) * 3 / 4; in virtqueue_enable_cb_delayed_packed()
1756 last_used_idx = READ_ONCE(vq->last_used_idx); in virtqueue_enable_cb_delayed_packed()
1760 if (used_idx >= vq->packed.vring.num) { in virtqueue_enable_cb_delayed_packed()
1761 used_idx -= vq->packed.vring.num; in virtqueue_enable_cb_delayed_packed()
1765 vq->packed.vring.driver->off_wrap = cpu_to_le16(used_idx | in virtqueue_enable_cb_delayed_packed()
1772 virtio_wmb(vq->weak_barriers); in virtqueue_enable_cb_delayed_packed()
1775 if (vq->packed.event_flags_shadow == VRING_PACKED_EVENT_FLAG_DISABLE) { in virtqueue_enable_cb_delayed_packed()
1776 vq->packed.event_flags_shadow = vq->event ? in virtqueue_enable_cb_delayed_packed()
1779 vq->packed.vring.driver->flags = in virtqueue_enable_cb_delayed_packed()
1780 cpu_to_le16(vq->packed.event_flags_shadow); in virtqueue_enable_cb_delayed_packed()
1787 virtio_mb(vq->weak_barriers); in virtqueue_enable_cb_delayed_packed()
1789 last_used_idx = READ_ONCE(vq->last_used_idx); in virtqueue_enable_cb_delayed_packed()
1792 if (is_used_desc_packed(vq, used_idx, wrap_counter)) { in virtqueue_enable_cb_delayed_packed()
1793 END_USE(vq); in virtqueue_enable_cb_delayed_packed()
1797 END_USE(vq); in virtqueue_enable_cb_delayed_packed()
1803 struct vring_virtqueue *vq = to_vvq(_vq); in virtqueue_detach_unused_buf_packed() local
1807 START_USE(vq); in virtqueue_detach_unused_buf_packed()
1809 for (i = 0; i < vq->packed.vring.num; i++) { in virtqueue_detach_unused_buf_packed()
1810 if (!vq->packed.desc_state[i].data) in virtqueue_detach_unused_buf_packed()
1813 buf = vq->packed.desc_state[i].data; in virtqueue_detach_unused_buf_packed()
1814 detach_buf_packed(vq, i, NULL); in virtqueue_detach_unused_buf_packed()
1815 END_USE(vq); in virtqueue_detach_unused_buf_packed()
1819 BUG_ON(vq->vq.num_free != vq->packed.vring.num); in virtqueue_detach_unused_buf_packed()
1821 END_USE(vq); in virtqueue_detach_unused_buf_packed()
1959 static void virtqueue_vring_attach_packed(struct vring_virtqueue *vq, in virtqueue_vring_attach_packed() argument
1962 vq->packed = *vring_packed; in virtqueue_vring_attach_packed()
1965 vq->free_head = 0; in virtqueue_vring_attach_packed()
1968 static void virtqueue_reinit_packed(struct vring_virtqueue *vq) in virtqueue_reinit_packed() argument
1970 memset(vq->packed.vring.device, 0, vq->packed.event_size_in_bytes); in virtqueue_reinit_packed()
1971 memset(vq->packed.vring.driver, 0, vq->packed.event_size_in_bytes); in virtqueue_reinit_packed()
1974 memset(vq->packed.vring.desc, 0, vq->packed.ring_size_in_bytes); in virtqueue_reinit_packed()
1976 virtqueue_init(vq, vq->packed.vring.num); in virtqueue_reinit_packed()
1977 virtqueue_vring_init_packed(&vq->packed, !!vq->vq.callback); in virtqueue_reinit_packed()
1993 struct vring_virtqueue *vq; in vring_create_virtqueue_packed() local
1999 vq = kmalloc(sizeof(*vq), GFP_KERNEL); in vring_create_virtqueue_packed()
2000 if (!vq) in vring_create_virtqueue_packed()
2003 vq->vq.callback = callback; in vring_create_virtqueue_packed()
2004 vq->vq.vdev = vdev; in vring_create_virtqueue_packed()
2005 vq->vq.name = name; in vring_create_virtqueue_packed()
2006 vq->vq.index = index; in vring_create_virtqueue_packed()
2007 vq->vq.reset = false; in vring_create_virtqueue_packed()
2008 vq->we_own_ring = true; in vring_create_virtqueue_packed()
2009 vq->notify = notify; in vring_create_virtqueue_packed()
2010 vq->weak_barriers = weak_barriers; in vring_create_virtqueue_packed()
2012 vq->broken = true; in vring_create_virtqueue_packed()
2014 vq->broken = false; in vring_create_virtqueue_packed()
2016 vq->packed_ring = true; in vring_create_virtqueue_packed()
2017 vq->use_dma_api = vring_use_dma_api(vdev); in vring_create_virtqueue_packed()
2019 vq->indirect = virtio_has_feature(vdev, VIRTIO_RING_F_INDIRECT_DESC) && in vring_create_virtqueue_packed()
2021 vq->event = virtio_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX); in vring_create_virtqueue_packed()
2024 vq->weak_barriers = false; in vring_create_virtqueue_packed()
2032 virtqueue_init(vq, num); in vring_create_virtqueue_packed()
2033 virtqueue_vring_attach_packed(vq, &vring_packed); in vring_create_virtqueue_packed()
2036 list_add_tail(&vq->vq.list, &vdev->vqs); in vring_create_virtqueue_packed()
2038 return &vq->vq; in vring_create_virtqueue_packed()
2041 kfree(vq); in vring_create_virtqueue_packed()
2051 struct vring_virtqueue *vq = to_vvq(_vq); in virtqueue_resize_packed() local
2062 vring_free(&vq->vq); in virtqueue_resize_packed()
2064 virtqueue_vring_init_packed(&vring_packed, !!vq->vq.callback); in virtqueue_resize_packed()
2066 virtqueue_init(vq, vring_packed.vring.num); in virtqueue_resize_packed()
2067 virtqueue_vring_attach_packed(vq, &vring_packed); in virtqueue_resize_packed()
2074 virtqueue_reinit_packed(vq); in virtqueue_resize_packed()
2092 struct vring_virtqueue *vq = to_vvq(_vq); in virtqueue_add() local
2094 return vq->packed_ring ? virtqueue_add_packed(_vq, sgs, total_sg, in virtqueue_add()
2148 int virtqueue_add_outbuf(struct virtqueue *vq, in virtqueue_add_outbuf() argument
2153 return virtqueue_add(vq, &sg, num, 1, 0, data, NULL, gfp); in virtqueue_add_outbuf()
2170 int virtqueue_add_inbuf(struct virtqueue *vq, in virtqueue_add_inbuf() argument
2175 return virtqueue_add(vq, &sg, num, 0, 1, data, NULL, gfp); in virtqueue_add_inbuf()
2193 int virtqueue_add_inbuf_ctx(struct virtqueue *vq, in virtqueue_add_inbuf_ctx() argument
2199 return virtqueue_add(vq, &sg, num, 0, 1, data, ctx, gfp); in virtqueue_add_inbuf_ctx()
2216 struct vring_virtqueue *vq = to_vvq(_vq); in virtqueue_kick_prepare() local
2218 return vq->packed_ring ? virtqueue_kick_prepare_packed(_vq) : in virtqueue_kick_prepare()
2233 struct vring_virtqueue *vq = to_vvq(_vq); in virtqueue_notify() local
2235 if (unlikely(vq->broken)) in virtqueue_notify()
2239 if (!vq->notify(_vq)) { in virtqueue_notify()
2240 vq->broken = true; in virtqueue_notify()
2259 bool virtqueue_kick(struct virtqueue *vq) in virtqueue_kick() argument
2261 if (virtqueue_kick_prepare(vq)) in virtqueue_kick()
2262 return virtqueue_notify(vq); in virtqueue_kick()
2287 struct vring_virtqueue *vq = to_vvq(_vq); in virtqueue_get_buf_ctx() local
2289 return vq->packed_ring ? virtqueue_get_buf_ctx_packed(_vq, len, ctx) : in virtqueue_get_buf_ctx()
2310 struct vring_virtqueue *vq = to_vvq(_vq); in virtqueue_disable_cb() local
2315 if (vq->event_triggered) in virtqueue_disable_cb()
2318 if (vq->packed_ring) in virtqueue_disable_cb()
2339 struct vring_virtqueue *vq = to_vvq(_vq); in virtqueue_enable_cb_prepare() local
2341 if (vq->event_triggered) in virtqueue_enable_cb_prepare()
2342 vq->event_triggered = false; in virtqueue_enable_cb_prepare()
2344 return vq->packed_ring ? virtqueue_enable_cb_prepare_packed(_vq) : in virtqueue_enable_cb_prepare()
2360 struct vring_virtqueue *vq = to_vvq(_vq); in virtqueue_poll() local
2362 if (unlikely(vq->broken)) in virtqueue_poll()
2365 virtio_mb(vq->weak_barriers); in virtqueue_poll()
2366 return vq->packed_ring ? virtqueue_poll_packed(_vq, last_used_idx) : in virtqueue_poll()
2405 struct vring_virtqueue *vq = to_vvq(_vq); in virtqueue_enable_cb_delayed() local
2407 if (vq->event_triggered) in virtqueue_enable_cb_delayed()
2408 vq->event_triggered = false; in virtqueue_enable_cb_delayed()
2410 return vq->packed_ring ? virtqueue_enable_cb_delayed_packed(_vq) : in virtqueue_enable_cb_delayed()
2425 struct vring_virtqueue *vq = to_vvq(_vq); in virtqueue_detach_unused_buf() local
2427 return vq->packed_ring ? virtqueue_detach_unused_buf_packed(_vq) : in virtqueue_detach_unused_buf()
2432 static inline bool more_used(const struct vring_virtqueue *vq) in more_used() argument
2434 return vq->packed_ring ? more_used_packed(vq) : more_used_split(vq); in more_used()
2447 struct vring_virtqueue *vq = to_vvq(_vq); in vring_interrupt() local
2449 if (!more_used(vq)) { in vring_interrupt()
2450 pr_debug("virtqueue interrupt with no work for %p\n", vq); in vring_interrupt()
2454 if (unlikely(vq->broken)) { in vring_interrupt()
2456 dev_warn_once(&vq->vq.vdev->dev, in vring_interrupt()
2465 if (vq->event) in vring_interrupt()
2466 vq->event_triggered = true; in vring_interrupt()
2468 pr_debug("virtqueue callback for %p (%p)\n", vq, vq->vq.callback); in vring_interrupt()
2469 if (vq->vq.callback) in vring_interrupt()
2470 vq->vq.callback(&vq->vq); in vring_interrupt()
2486 struct vring_virtqueue *vq; in __vring_new_virtqueue() local
2492 vq = kmalloc(sizeof(*vq), GFP_KERNEL); in __vring_new_virtqueue()
2493 if (!vq) in __vring_new_virtqueue()
2496 vq->packed_ring = false; in __vring_new_virtqueue()
2497 vq->vq.callback = callback; in __vring_new_virtqueue()
2498 vq->vq.vdev = vdev; in __vring_new_virtqueue()
2499 vq->vq.name = name; in __vring_new_virtqueue()
2500 vq->vq.index = index; in __vring_new_virtqueue()
2501 vq->vq.reset = false; in __vring_new_virtqueue()
2502 vq->we_own_ring = false; in __vring_new_virtqueue()
2503 vq->notify = notify; in __vring_new_virtqueue()
2504 vq->weak_barriers = weak_barriers; in __vring_new_virtqueue()
2506 vq->broken = true; in __vring_new_virtqueue()
2508 vq->broken = false; in __vring_new_virtqueue()
2510 vq->use_dma_api = vring_use_dma_api(vdev); in __vring_new_virtqueue()
2512 vq->indirect = virtio_has_feature(vdev, VIRTIO_RING_F_INDIRECT_DESC) && in __vring_new_virtqueue()
2514 vq->event = virtio_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX); in __vring_new_virtqueue()
2517 vq->weak_barriers = false; in __vring_new_virtqueue()
2521 kfree(vq); in __vring_new_virtqueue()
2525 virtqueue_vring_init_split(vring_split, vq); in __vring_new_virtqueue()
2527 virtqueue_init(vq, vring_split->vring.num); in __vring_new_virtqueue()
2528 virtqueue_vring_attach_split(vq, vring_split); in __vring_new_virtqueue()
2531 list_add_tail(&vq->vq.list, &vdev->vqs); in __vring_new_virtqueue()
2533 return &vq->vq; in __vring_new_virtqueue()
2585 void (*recycle)(struct virtqueue *vq, void *buf)) in virtqueue_resize() argument
2587 struct vring_virtqueue *vq = to_vvq(_vq); in virtqueue_resize() local
2588 struct virtio_device *vdev = vq->vq.vdev; in virtqueue_resize()
2592 if (!vq->we_own_ring) in virtqueue_resize()
2595 if (num > vq->vq.num_max) in virtqueue_resize()
2601 if ((vq->packed_ring ? vq->packed.vring.num : vq->split.vring.num) == num) in virtqueue_resize()
2617 if (vq->packed_ring) in virtqueue_resize()
2637 bool (*notify)(struct virtqueue *vq), in vring_new_virtqueue() argument
2638 void (*callback)(struct virtqueue *vq), in vring_new_virtqueue() argument
2654 struct vring_virtqueue *vq = to_vvq(_vq); in vring_free() local
2656 if (vq->we_own_ring) { in vring_free()
2657 if (vq->packed_ring) { in vring_free()
2658 vring_free_queue(vq->vq.vdev, in vring_free()
2659 vq->packed.ring_size_in_bytes, in vring_free()
2660 vq->packed.vring.desc, in vring_free()
2661 vq->packed.ring_dma_addr); in vring_free()
2663 vring_free_queue(vq->vq.vdev, in vring_free()
2664 vq->packed.event_size_in_bytes, in vring_free()
2665 vq->packed.vring.driver, in vring_free()
2666 vq->packed.driver_event_dma_addr); in vring_free()
2668 vring_free_queue(vq->vq.vdev, in vring_free()
2669 vq->packed.event_size_in_bytes, in vring_free()
2670 vq->packed.vring.device, in vring_free()
2671 vq->packed.device_event_dma_addr); in vring_free()
2673 kfree(vq->packed.desc_state); in vring_free()
2674 kfree(vq->packed.desc_extra); in vring_free()
2676 vring_free_queue(vq->vq.vdev, in vring_free()
2677 vq->split.queue_size_in_bytes, in vring_free()
2678 vq->split.vring.desc, in vring_free()
2679 vq->split.queue_dma_addr); in vring_free()
2682 if (!vq->packed_ring) { in vring_free()
2683 kfree(vq->split.desc_state); in vring_free()
2684 kfree(vq->split.desc_extra); in vring_free()
2690 struct vring_virtqueue *vq = to_vvq(_vq); in vring_del_virtqueue() local
2692 spin_lock(&vq->vq.vdev->vqs_list_lock); in vring_del_virtqueue()
2694 spin_unlock(&vq->vq.vdev->vqs_list_lock); in vring_del_virtqueue()
2698 kfree(vq); in vring_del_virtqueue()
2739 struct vring_virtqueue *vq = to_vvq(_vq); in virtqueue_get_vring_size() local
2741 return vq->packed_ring ? vq->packed.vring.num : vq->split.vring.num; in virtqueue_get_vring_size()
2750 struct vring_virtqueue *vq = to_vvq(_vq); in __virtqueue_break() local
2753 WRITE_ONCE(vq->broken, true); in __virtqueue_break()
2762 struct vring_virtqueue *vq = to_vvq(_vq); in __virtqueue_unbreak() local
2765 WRITE_ONCE(vq->broken, false); in __virtqueue_unbreak()
2771 struct vring_virtqueue *vq = to_vvq(_vq); in virtqueue_is_broken() local
2773 return READ_ONCE(vq->broken); in virtqueue_is_broken()
2787 struct vring_virtqueue *vq = to_vvq(_vq); in virtio_break_device() local
2790 WRITE_ONCE(vq->broken, true); in virtio_break_device()
2809 struct vring_virtqueue *vq = to_vvq(_vq); in __virtio_unbreak_device() local
2812 WRITE_ONCE(vq->broken, false); in __virtio_unbreak_device()
2820 struct vring_virtqueue *vq = to_vvq(_vq); in virtqueue_get_desc_addr() local
2822 BUG_ON(!vq->we_own_ring); in virtqueue_get_desc_addr()
2824 if (vq->packed_ring) in virtqueue_get_desc_addr()
2825 return vq->packed.ring_dma_addr; in virtqueue_get_desc_addr()
2827 return vq->split.queue_dma_addr; in virtqueue_get_desc_addr()
2833 struct vring_virtqueue *vq = to_vvq(_vq); in virtqueue_get_avail_addr() local
2835 BUG_ON(!vq->we_own_ring); in virtqueue_get_avail_addr()
2837 if (vq->packed_ring) in virtqueue_get_avail_addr()
2838 return vq->packed.driver_event_dma_addr; in virtqueue_get_avail_addr()
2840 return vq->split.queue_dma_addr + in virtqueue_get_avail_addr()
2841 ((char *)vq->split.vring.avail - (char *)vq->split.vring.desc); in virtqueue_get_avail_addr()
2847 struct vring_virtqueue *vq = to_vvq(_vq); in virtqueue_get_used_addr() local
2849 BUG_ON(!vq->we_own_ring); in virtqueue_get_used_addr()
2851 if (vq->packed_ring) in virtqueue_get_used_addr()
2852 return vq->packed.device_event_dma_addr; in virtqueue_get_used_addr()
2854 return vq->split.queue_dma_addr + in virtqueue_get_used_addr()
2855 ((char *)vq->split.vring.used - (char *)vq->split.vring.desc); in virtqueue_get_used_addr()
2860 const struct vring *virtqueue_get_vring(struct virtqueue *vq) in virtqueue_get_vring() argument
2862 return &to_vvq(vq)->split.vring; in virtqueue_get_vring()