26 /* Vring size. */
63  * @num: vring size (number of descriptors)
64  * @align: vring alignment size
65  * @index: vring index
66  * @vdev_id: vring virtio id (VIRTIO_ID_xxx)
151  * @vring: Tx/Rx ring
166 	struct mlxbf_tmfifo_vring *vring[2];  member
211 	struct mlxbf_tmfifo_vring *vring;  in mlxbf_tmfifo_free_vrings()  local
215 		vring = &tm_vdev->vrings[i];  in mlxbf_tmfifo_free_vrings()
216 		if (vring->va) {  in mlxbf_tmfifo_free_vrings()
217 			size = vring_size(vring->num, vring->align);  in mlxbf_tmfifo_free_vrings()
219 					  vring->va, vring->dma);  in mlxbf_tmfifo_free_vrings()
220 			vring->va = NULL;  in mlxbf_tmfifo_free_vrings()
221 			if (vring->vq) {  in mlxbf_tmfifo_free_vrings()
222 				vring_del_virtqueue(vring->vq);  in mlxbf_tmfifo_free_vrings()
223 				vring->vq = NULL;  in mlxbf_tmfifo_free_vrings()
233 	struct mlxbf_tmfifo_vring *vring;  in mlxbf_tmfifo_alloc_vrings()  local
240 		vring = &tm_vdev->vrings[i];  in mlxbf_tmfifo_alloc_vrings()
241 		vring->fifo = fifo;  in mlxbf_tmfifo_alloc_vrings()
242 		vring->num = MLXBF_TMFIFO_VRING_SIZE;  in mlxbf_tmfifo_alloc_vrings()
243 		vring->align = SMP_CACHE_BYTES;  in mlxbf_tmfifo_alloc_vrings()
244 		vring->index = i;  in mlxbf_tmfifo_alloc_vrings()
245 		vring->vdev_id = tm_vdev->vdev.id.device;  in mlxbf_tmfifo_alloc_vrings()
248 		size = vring_size(vring->num, vring->align);  in mlxbf_tmfifo_alloc_vrings()
256 		vring->va = va;  in mlxbf_tmfifo_alloc_vrings()
257 		vring->dma = dma;  in mlxbf_tmfifo_alloc_vrings()
286 /* Get the next packet descriptor from the vring. */
288 mlxbf_tmfifo_get_next_desc(struct mlxbf_tmfifo_vring *vring)  in mlxbf_tmfifo_get_next_desc()  argument
290 	const struct vring *vr = virtqueue_get_vring(vring->vq);  in mlxbf_tmfifo_get_next_desc()
291 	struct virtio_device *vdev = vring->vq->vdev;  in mlxbf_tmfifo_get_next_desc()
294 	if (vring->next_avail == virtio16_to_cpu(vdev, vr->avail->idx))  in mlxbf_tmfifo_get_next_desc()
300 	idx = vring->next_avail % vr->num;  in mlxbf_tmfifo_get_next_desc()
305 	vring->next_avail++;  in mlxbf_tmfifo_get_next_desc()
311 static void mlxbf_tmfifo_release_desc(struct mlxbf_tmfifo_vring *vring,  in mlxbf_tmfifo_release_desc()  argument
314 	const struct vring *vr = virtqueue_get_vring(vring->vq);  in mlxbf_tmfifo_release_desc()
315 	struct virtio_device *vdev = vring->vq->vdev;  in mlxbf_tmfifo_release_desc()
333 static u32 mlxbf_tmfifo_get_pkt_len(struct mlxbf_tmfifo_vring *vring,  in mlxbf_tmfifo_get_pkt_len()  argument
336 	const struct vring *vr = virtqueue_get_vring(vring->vq);  in mlxbf_tmfifo_get_pkt_len()
337 	struct virtio_device *vdev = vring->vq->vdev;  in mlxbf_tmfifo_get_pkt_len()
351 static void mlxbf_tmfifo_release_pending_pkt(struct mlxbf_tmfifo_vring *vring)  in mlxbf_tmfifo_release_pending_pkt()  argument
356 	if (vring->desc_head) {  in mlxbf_tmfifo_release_pending_pkt()
357 		desc_head = vring->desc_head;  in mlxbf_tmfifo_release_pending_pkt()
358 		len = vring->pkt_len;  in mlxbf_tmfifo_release_pending_pkt()
360 		desc_head = mlxbf_tmfifo_get_next_desc(vring);  in mlxbf_tmfifo_release_pending_pkt()
361 		len = mlxbf_tmfifo_get_pkt_len(vring, desc_head);  in mlxbf_tmfifo_release_pending_pkt()
365 		mlxbf_tmfifo_release_desc(vring, desc_head, len);  in mlxbf_tmfifo_release_pending_pkt()
367 	vring->pkt_len = 0;  in mlxbf_tmfifo_release_pending_pkt()
368 	vring->desc = NULL;  in mlxbf_tmfifo_release_pending_pkt()
369 	vring->desc_head = NULL;  in mlxbf_tmfifo_release_pending_pkt()
372 static void mlxbf_tmfifo_init_net_desc(struct mlxbf_tmfifo_vring *vring,  in mlxbf_tmfifo_init_net_desc()  argument
375 	struct virtio_device *vdev = vring->vq->vdev;  in mlxbf_tmfifo_init_net_desc()
384 mlxbf_tmfifo_get_next_pkt(struct mlxbf_tmfifo_vring *vring, bool is_rx)  in mlxbf_tmfifo_get_next_pkt()  argument
388 	desc = mlxbf_tmfifo_get_next_desc(vring);  in mlxbf_tmfifo_get_next_pkt()
389 	if (desc && is_rx && vring->vdev_id == VIRTIO_ID_NET)  in mlxbf_tmfifo_get_next_pkt()
390 		mlxbf_tmfifo_init_net_desc(vring, desc, is_rx);  in mlxbf_tmfifo_get_next_pkt()
392 	vring->desc_head = desc;  in mlxbf_tmfifo_get_next_pkt()
393 	vring->desc = desc;  in mlxbf_tmfifo_get_next_pkt()
415 					    struct mlxbf_tmfifo_vring *vring,  in mlxbf_tmfifo_console_output_one()  argument
418 	const struct vring *vr = virtqueue_get_vring(vring->vq);  in mlxbf_tmfifo_console_output_one()
448 					struct mlxbf_tmfifo_vring *vring)  in mlxbf_tmfifo_console_output()  argument
453 	desc = mlxbf_tmfifo_get_next_desc(vring);  in mlxbf_tmfifo_console_output()
456 		len = mlxbf_tmfifo_get_pkt_len(vring, desc);  in mlxbf_tmfifo_console_output()
460 			mlxbf_tmfifo_release_desc(vring, desc, len);  in mlxbf_tmfifo_console_output()
464 		mlxbf_tmfifo_console_output_one(cons, vring, desc);  in mlxbf_tmfifo_console_output()
465 		mlxbf_tmfifo_release_desc(vring, desc, len);  in mlxbf_tmfifo_console_output()
466 		desc = mlxbf_tmfifo_get_next_desc(vring);  in mlxbf_tmfifo_console_output()
562 static void mlxbf_tmfifo_rxtx_word(struct mlxbf_tmfifo_vring *vring,  in mlxbf_tmfifo_rxtx_word()  argument
566 	struct virtio_device *vdev = vring->vq->vdev;  in mlxbf_tmfifo_rxtx_word()
567 	struct mlxbf_tmfifo *fifo = vring->fifo;  in mlxbf_tmfifo_rxtx_word()
578 	if (vring->cur_len + sizeof(u64) <= len) {  in mlxbf_tmfifo_rxtx_word()
581 			memcpy(addr + vring->cur_len, &data, sizeof(u64));  in mlxbf_tmfifo_rxtx_word()
583 			memcpy(&data, addr + vring->cur_len, sizeof(u64));  in mlxbf_tmfifo_rxtx_word()
584 		vring->cur_len += sizeof(u64);  in mlxbf_tmfifo_rxtx_word()
588 			memcpy(addr + vring->cur_len, &data,  in mlxbf_tmfifo_rxtx_word()
589 			       len - vring->cur_len);  in mlxbf_tmfifo_rxtx_word()
591 			memcpy(&data, addr + vring->cur_len,  in mlxbf_tmfifo_rxtx_word()
592 			       len - vring->cur_len);  in mlxbf_tmfifo_rxtx_word()
593 		vring->cur_len = len;  in mlxbf_tmfifo_rxtx_word()
604  * In Rx case, the packet might be found to belong to a different vring since
608 static void mlxbf_tmfifo_rxtx_header(struct mlxbf_tmfifo_vring *vring,  in mlxbf_tmfifo_rxtx_header()  argument
612 	struct mlxbf_tmfifo *fifo = vring->fifo;  in mlxbf_tmfifo_rxtx_header()
643 		 * Check whether the new packet still belongs to this vring.  in mlxbf_tmfifo_rxtx_header()
644 		 * If not, update the pkt_len of the new vring.  in mlxbf_tmfifo_rxtx_header()
646 		if (vdev_id != vring->vdev_id) {  in mlxbf_tmfifo_rxtx_header()
651 			vring->desc = desc;  in mlxbf_tmfifo_rxtx_header()
652 			vring = &tm_dev2->vrings[MLXBF_TMFIFO_VRING_RX];  in mlxbf_tmfifo_rxtx_header()
655 		vring->pkt_len = ntohs(hdr.len) + hdr_len;  in mlxbf_tmfifo_rxtx_header()
658 		hdr_len = (vring->vdev_id == VIRTIO_ID_NET) ?  in mlxbf_tmfifo_rxtx_header()
660 		vring->pkt_len = mlxbf_tmfifo_get_pkt_len(vring, desc);  in mlxbf_tmfifo_rxtx_header()
661 		hdr.type = (vring->vdev_id == VIRTIO_ID_NET) ?  in mlxbf_tmfifo_rxtx_header()
663 		hdr.len = htons(vring->pkt_len - hdr_len);  in mlxbf_tmfifo_rxtx_header()
667 	vring->cur_len = hdr_len;  in mlxbf_tmfifo_rxtx_header()
668 	vring->rem_len = vring->pkt_len;  in mlxbf_tmfifo_rxtx_header()
669 	fifo->vring[is_rx] = vring;  in mlxbf_tmfifo_rxtx_header()
677 static bool mlxbf_tmfifo_rxtx_one_desc(struct mlxbf_tmfifo_vring *vring,  in mlxbf_tmfifo_rxtx_one_desc()  argument
680 	const struct vring *vr = virtqueue_get_vring(vring->vq);  in mlxbf_tmfifo_rxtx_one_desc()
681 	struct mlxbf_tmfifo *fifo = vring->fifo;  in mlxbf_tmfifo_rxtx_one_desc()
688 	vdev = &fifo->vdev[vring->vdev_id]->vdev;  in mlxbf_tmfifo_rxtx_one_desc()
691 	if (!vring->desc) {  in mlxbf_tmfifo_rxtx_one_desc()
692 		desc = mlxbf_tmfifo_get_next_pkt(vring, is_rx);  in mlxbf_tmfifo_rxtx_one_desc()
696 		desc = vring->desc;  in mlxbf_tmfifo_rxtx_one_desc()
700 	if (vring->pkt_len == 0) {  in mlxbf_tmfifo_rxtx_one_desc()
701 		mlxbf_tmfifo_rxtx_header(vring, desc, is_rx, &vring_change);  in mlxbf_tmfifo_rxtx_one_desc()
712 	if (len > vring->rem_len)  in mlxbf_tmfifo_rxtx_one_desc()
713 		len = vring->rem_len;  in mlxbf_tmfifo_rxtx_one_desc()
716 	if (vring->cur_len < len) {  in mlxbf_tmfifo_rxtx_one_desc()
717 		mlxbf_tmfifo_rxtx_word(vring, desc, is_rx, len);  in mlxbf_tmfifo_rxtx_one_desc()
722 	if (vring->cur_len == len) {  in mlxbf_tmfifo_rxtx_one_desc()
723 		vring->cur_len = 0;  in mlxbf_tmfifo_rxtx_one_desc()
724 		vring->rem_len -= len;  in mlxbf_tmfifo_rxtx_one_desc()
727 		if (vring->rem_len > 0 &&  in mlxbf_tmfifo_rxtx_one_desc()
735 		mlxbf_tmfifo_release_pending_pkt(vring);  in mlxbf_tmfifo_rxtx_one_desc()
737 		fifo->vring[is_rx] = NULL;  in mlxbf_tmfifo_rxtx_one_desc()
747 		vring_interrupt(0, vring->vq);  in mlxbf_tmfifo_rxtx_one_desc()
753 	vring->desc = desc;  in mlxbf_tmfifo_rxtx_one_desc()
759 static void mlxbf_tmfifo_rxtx(struct mlxbf_tmfifo_vring *vring, bool is_rx)  in mlxbf_tmfifo_rxtx()  argument
761 	int avail = 0, devid = vring->vdev_id;  in mlxbf_tmfifo_rxtx()
765 	fifo = vring->fifo;  in mlxbf_tmfifo_rxtx()
771 	/* Return if another vring is running. */  in mlxbf_tmfifo_rxtx()
772 	if (fifo->vring[is_rx] && fifo->vring[is_rx] != vring)  in mlxbf_tmfifo_rxtx()
797 		more = mlxbf_tmfifo_rxtx_one_desc(vring, is_rx, &avail);  in mlxbf_tmfifo_rxtx()
806 	struct mlxbf_tmfifo_vring *vring;  in mlxbf_tmfifo_work_rxtx()  local
816 			vring = &tm_vdev->vrings[queue_id];  in mlxbf_tmfifo_work_rxtx()
817 			if (vring->vq)  in mlxbf_tmfifo_work_rxtx()
818 				mlxbf_tmfifo_rxtx(vring, is_rx);  in mlxbf_tmfifo_work_rxtx()
848 	struct mlxbf_tmfifo_vring *vring = vq->priv;  in mlxbf_tmfifo_virtio_notify()  local
853 	fifo = vring->fifo;  in mlxbf_tmfifo_virtio_notify()
859 	if (vring->index & BIT(0)) {  in mlxbf_tmfifo_virtio_notify()
862 		 * In such case, the vring needs to be served right away. For  in mlxbf_tmfifo_virtio_notify()
866 		if (vring->vdev_id == VIRTIO_ID_CONSOLE) {  in mlxbf_tmfifo_virtio_notify()
869 			mlxbf_tmfifo_console_output(tm_vdev, vring);  in mlxbf_tmfifo_virtio_notify()
907 	struct mlxbf_tmfifo_vring *vring;  in mlxbf_tmfifo_virtio_del_vqs()  local
912 		vring = &tm_vdev->vrings[i];  in mlxbf_tmfifo_virtio_del_vqs()
915 		if (vring->desc)  in mlxbf_tmfifo_virtio_del_vqs()
916 			mlxbf_tmfifo_release_pending_pkt(vring);  in mlxbf_tmfifo_virtio_del_vqs()
917 		vq = vring->vq;  in mlxbf_tmfifo_virtio_del_vqs()
919 			vring->vq = NULL;  in mlxbf_tmfifo_virtio_del_vqs()
935 	struct mlxbf_tmfifo_vring *vring;  in mlxbf_tmfifo_virtio_find_vqs()  local
947 		vring = &tm_vdev->vrings[i];  in mlxbf_tmfifo_virtio_find_vqs()
949 		/* zero vring */  in mlxbf_tmfifo_virtio_find_vqs()
950 		size = vring_size(vring->num, vring->align);  in mlxbf_tmfifo_virtio_find_vqs()
951 		memset(vring->va, 0, size);  in mlxbf_tmfifo_virtio_find_vqs()
952 		vq = vring_new_virtqueue(i, vring->num, vring->align, vdev,  in mlxbf_tmfifo_virtio_find_vqs()
953 					 false, false, vring->va,  in mlxbf_tmfifo_virtio_find_vqs()
962 		vq->num_max = vring->num;  in mlxbf_tmfifo_virtio_find_vqs()
965 		vring->vq = vq;  in mlxbf_tmfifo_virtio_find_vqs()
966 		vq->priv = vring;  in mlxbf_tmfifo_virtio_find_vqs()
1084 		dev_err(dev, "unable to allocate vring\n");  in mlxbf_tmfifo_create_vdev()