92 	struct virtio_gpu_vbuffer *vbuf;  in virtio_gpu_get_vbuf()  local
94 	vbuf = kmem_cache_zalloc(vgdev->vbufs, GFP_KERNEL);  in virtio_gpu_get_vbuf()
95 	if (!vbuf)  in virtio_gpu_get_vbuf()
100 	vbuf->buf = (void *)vbuf + sizeof(*vbuf);  in virtio_gpu_get_vbuf()
101 	vbuf->size = size;  in virtio_gpu_get_vbuf()
103 	vbuf->resp_cb = resp_cb;  in virtio_gpu_get_vbuf()
104 	vbuf->resp_size = resp_size;  in virtio_gpu_get_vbuf()
106 		vbuf->resp_buf = (void *)vbuf->buf + size;  in virtio_gpu_get_vbuf()
108 		vbuf->resp_buf = resp_buf;  in virtio_gpu_get_vbuf()
109 	BUG_ON(!vbuf->resp_buf);  in virtio_gpu_get_vbuf()
110 	return vbuf;  in virtio_gpu_get_vbuf()
114 virtio_gpu_vbuf_ctrl_hdr(struct virtio_gpu_vbuffer *vbuf)  in virtio_gpu_vbuf_ctrl_hdr()  argument
120 	return (struct virtio_gpu_ctrl_hdr *)vbuf->buf;  in virtio_gpu_vbuf_ctrl_hdr()
127 	struct virtio_gpu_vbuffer *vbuf;  in virtio_gpu_alloc_cursor()  local
129 	vbuf = virtio_gpu_get_vbuf  in virtio_gpu_alloc_cursor()
132 	if (IS_ERR(vbuf)) {  in virtio_gpu_alloc_cursor()
134 		return ERR_CAST(vbuf);  in virtio_gpu_alloc_cursor()
136 	*vbuffer_p = vbuf;  in virtio_gpu_alloc_cursor()
137 	return (struct virtio_gpu_update_cursor *)vbuf->buf;  in virtio_gpu_alloc_cursor()
146 	struct virtio_gpu_vbuffer *vbuf;  in virtio_gpu_alloc_cmd_resp()  local
148 	vbuf = virtio_gpu_get_vbuf(vgdev, cmd_size,  in virtio_gpu_alloc_cmd_resp()
150 	if (IS_ERR(vbuf)) {  in virtio_gpu_alloc_cmd_resp()
152 		return ERR_CAST(vbuf);  in virtio_gpu_alloc_cmd_resp()
154 	*vbuffer_p = vbuf;  in virtio_gpu_alloc_cmd_resp()
155 	return (struct virtio_gpu_command *)vbuf->buf;  in virtio_gpu_alloc_cmd_resp()
178 		      struct virtio_gpu_vbuffer *vbuf)  in free_vbuf()  argument
180 	if (vbuf->resp_size > MAX_INLINE_RESP_SIZE)  in free_vbuf()
181 		kfree(vbuf->resp_buf);  in free_vbuf()
182 	kvfree(vbuf->data_buf);  in free_vbuf()
183 	kmem_cache_free(vgdev->vbufs, vbuf);  in free_vbuf()
188 	struct virtio_gpu_vbuffer *vbuf;  in reclaim_vbufs()  local
192 	while ((vbuf = virtqueue_get_buf(vq, &len))) {  in reclaim_vbufs()
193 		list_add_tail(&vbuf->list, reclaim_list);  in reclaim_vbufs()
324 				     struct virtio_gpu_vbuffer *vbuf,  in virtio_gpu_queue_ctrl_sgs()  argument
335 		if (fence && vbuf->objs)  in virtio_gpu_queue_ctrl_sgs()
336 			virtio_gpu_array_unlock_resv(vbuf->objs);  in virtio_gpu_queue_ctrl_sgs()
337 		free_vbuf(vgdev, vbuf);  in virtio_gpu_queue_ctrl_sgs()
358 		virtio_gpu_fence_emit(vgdev, virtio_gpu_vbuf_ctrl_hdr(vbuf),  in virtio_gpu_queue_ctrl_sgs()
360 		if (vbuf->objs) {  in virtio_gpu_queue_ctrl_sgs()
361 			virtio_gpu_array_add_fence(vbuf->objs, &fence->f);  in virtio_gpu_queue_ctrl_sgs()
362 			virtio_gpu_array_unlock_resv(vbuf->objs);  in virtio_gpu_queue_ctrl_sgs()
366 	ret = virtqueue_add_sgs(vq, sgs, outcnt, incnt, vbuf, GFP_ATOMIC);  in virtio_gpu_queue_ctrl_sgs()
369 	trace_virtio_gpu_cmd_queue(vq, virtio_gpu_vbuf_ctrl_hdr(vbuf));  in virtio_gpu_queue_ctrl_sgs()
380 					       struct virtio_gpu_vbuffer *vbuf,  in virtio_gpu_queue_fenced_ctrl_buffer()  argument
388 	sg_init_one(&vcmd, vbuf->buf, vbuf->size);  in virtio_gpu_queue_fenced_ctrl_buffer()
394 	if (vbuf->data_size) {  in virtio_gpu_queue_fenced_ctrl_buffer()
395 		if (is_vmalloc_addr(vbuf->data_buf)) {  in virtio_gpu_queue_fenced_ctrl_buffer()
397 			sgt = vmalloc_to_sgt(vbuf->data_buf, vbuf->data_size,  in virtio_gpu_queue_fenced_ctrl_buffer()
400 				if (fence && vbuf->objs)  in virtio_gpu_queue_fenced_ctrl_buffer()
401 					virtio_gpu_array_unlock_resv(vbuf->objs);  in virtio_gpu_queue_fenced_ctrl_buffer()
408 			sg_init_one(&vout, vbuf->data_buf, vbuf->data_size);  in virtio_gpu_queue_fenced_ctrl_buffer()
416 	if (vbuf->resp_size) {  in virtio_gpu_queue_fenced_ctrl_buffer()
417 		sg_init_one(&vresp, vbuf->resp_buf, vbuf->resp_size);  in virtio_gpu_queue_fenced_ctrl_buffer()
423 	ret = virtio_gpu_queue_ctrl_sgs(vgdev, vbuf, fence, elemcnt, sgs, outcnt,  in virtio_gpu_queue_fenced_ctrl_buffer()
450 					struct virtio_gpu_vbuffer *vbuf)  in virtio_gpu_queue_ctrl_buffer()  argument
452 	return virtio_gpu_queue_fenced_ctrl_buffer(vgdev, vbuf, NULL);  in virtio_gpu_queue_ctrl_buffer()
456 				    struct virtio_gpu_vbuffer *vbuf)  in virtio_gpu_queue_cursor()  argument
464 		free_vbuf(vgdev, vbuf);  in virtio_gpu_queue_cursor()
468 	sg_init_one(&ccmd, vbuf->buf, vbuf->size);  in virtio_gpu_queue_cursor()
474 	ret = virtqueue_add_sgs(vq, sgs, outcnt, 0, vbuf, GFP_ATOMIC);  in virtio_gpu_queue_cursor()
482 			virtio_gpu_vbuf_ctrl_hdr(vbuf));  in virtio_gpu_queue_cursor()
507 	struct virtio_gpu_vbuffer *vbuf;  in virtio_gpu_cmd_create_resource()  local
509 	cmd_p = virtio_gpu_alloc_cmd(vgdev, &vbuf, sizeof(*cmd_p));  in virtio_gpu_cmd_create_resource()
511 	vbuf->objs = objs;  in virtio_gpu_cmd_create_resource()
519 	virtio_gpu_queue_fenced_ctrl_buffer(vgdev, vbuf, fence);  in virtio_gpu_cmd_create_resource()
524 				    struct virtio_gpu_vbuffer *vbuf)  in virtio_gpu_cmd_unref_cb()  argument
528 	bo = vbuf->resp_cb_data;  in virtio_gpu_cmd_unref_cb()
529 	vbuf->resp_cb_data = NULL;  in virtio_gpu_cmd_unref_cb()
538 	struct virtio_gpu_vbuffer *vbuf;  in virtio_gpu_cmd_unref_resource()  local
541 	cmd_p = virtio_gpu_alloc_cmd_cb(vgdev, &vbuf, sizeof(*cmd_p),  in virtio_gpu_cmd_unref_resource()
548 	vbuf->resp_cb_data = bo;  in virtio_gpu_cmd_unref_resource()
549 	ret = virtio_gpu_queue_ctrl_buffer(vgdev, vbuf);  in virtio_gpu_cmd_unref_resource()
560 	struct virtio_gpu_vbuffer *vbuf;  in virtio_gpu_cmd_set_scanout()  local
562 	cmd_p = virtio_gpu_alloc_cmd(vgdev, &vbuf, sizeof(*cmd_p));  in virtio_gpu_cmd_set_scanout()
573 	virtio_gpu_queue_ctrl_buffer(vgdev, vbuf);  in virtio_gpu_cmd_set_scanout()
582 	struct virtio_gpu_vbuffer *vbuf;  in virtio_gpu_cmd_resource_flush()  local
584 	cmd_p = virtio_gpu_alloc_cmd(vgdev, &vbuf, sizeof(*cmd_p));  in virtio_gpu_cmd_resource_flush()
594 	virtio_gpu_queue_ctrl_buffer(vgdev, vbuf);  in virtio_gpu_cmd_resource_flush()
606 	struct virtio_gpu_vbuffer *vbuf;  in virtio_gpu_cmd_transfer_to_host_2d()  local
614 	cmd_p = virtio_gpu_alloc_cmd(vgdev, &vbuf, sizeof(*cmd_p));  in virtio_gpu_cmd_transfer_to_host_2d()
616 	vbuf->objs = objs;  in virtio_gpu_cmd_transfer_to_host_2d()
626 	virtio_gpu_queue_fenced_ctrl_buffer(vgdev, vbuf, fence);  in virtio_gpu_cmd_transfer_to_host_2d()
637 	struct virtio_gpu_vbuffer *vbuf;  in virtio_gpu_cmd_resource_attach_backing()  local
639 	cmd_p = virtio_gpu_alloc_cmd(vgdev, &vbuf, sizeof(*cmd_p));  in virtio_gpu_cmd_resource_attach_backing()
646 	vbuf->data_buf = ents;  in virtio_gpu_cmd_resource_attach_backing()
647 	vbuf->data_size = sizeof(*ents) * nents;  in virtio_gpu_cmd_resource_attach_backing()
649 	virtio_gpu_queue_fenced_ctrl_buffer(vgdev, vbuf, fence);  in virtio_gpu_cmd_resource_attach_backing()
653 					       struct virtio_gpu_vbuffer *vbuf)  in virtio_gpu_cmd_get_display_info_cb()  argument
656 		(struct virtio_gpu_resp_display_info *)vbuf->resp_buf;  in virtio_gpu_cmd_get_display_info_cb()
682 					      struct virtio_gpu_vbuffer *vbuf)  in virtio_gpu_cmd_get_capset_info_cb()  argument
685 		(struct virtio_gpu_get_capset_info *)vbuf->buf;  in virtio_gpu_cmd_get_capset_info_cb()
687 		(struct virtio_gpu_resp_capset_info *)vbuf->resp_buf;  in virtio_gpu_cmd_get_capset_info_cb()
703 				     struct virtio_gpu_vbuffer *vbuf)  in virtio_gpu_cmd_capset_cb()  argument
706 		(struct virtio_gpu_get_capset *)vbuf->buf;  in virtio_gpu_cmd_capset_cb()
708 		(struct virtio_gpu_resp_capset *)vbuf->resp_buf;  in virtio_gpu_cmd_capset_cb()
740 				       struct virtio_gpu_vbuffer *vbuf)  in virtio_gpu_cmd_get_edid_cb()  argument
743 		(struct virtio_gpu_cmd_get_edid *)vbuf->buf;  in virtio_gpu_cmd_get_edid_cb()
745 		(struct virtio_gpu_resp_edid *)vbuf->resp_buf;  in virtio_gpu_cmd_get_edid_cb()
769 	struct virtio_gpu_vbuffer *vbuf;  in virtio_gpu_cmd_get_display_info()  local
778 		(vgdev, &virtio_gpu_cmd_get_display_info_cb, &vbuf,  in virtio_gpu_cmd_get_display_info()
785 	virtio_gpu_queue_ctrl_buffer(vgdev, vbuf);  in virtio_gpu_cmd_get_display_info()
792 	struct virtio_gpu_vbuffer *vbuf;  in virtio_gpu_cmd_get_capset_info()  local
801 		(vgdev, &virtio_gpu_cmd_get_capset_info_cb, &vbuf,  in virtio_gpu_cmd_get_capset_info()
808 	virtio_gpu_queue_ctrl_buffer(vgdev, vbuf);  in virtio_gpu_cmd_get_capset_info()
817 	struct virtio_gpu_vbuffer *vbuf;  in virtio_gpu_cmd_get_capset()  local
876 		(vgdev, &virtio_gpu_cmd_capset_cb, &vbuf, sizeof(*cmd_p),  in virtio_gpu_cmd_get_capset()
883 	virtio_gpu_queue_ctrl_buffer(vgdev, vbuf);  in virtio_gpu_cmd_get_capset()
891 	struct virtio_gpu_vbuffer *vbuf;  in virtio_gpu_cmd_get_edids()  local
905 			(vgdev, &virtio_gpu_cmd_get_edid_cb, &vbuf,  in virtio_gpu_cmd_get_edids()
910 		virtio_gpu_queue_ctrl_buffer(vgdev, vbuf);  in virtio_gpu_cmd_get_edids()
920 	struct virtio_gpu_vbuffer *vbuf;  in virtio_gpu_cmd_context_create()  local
922 	cmd_p = virtio_gpu_alloc_cmd(vgdev, &vbuf, sizeof(*cmd_p));  in virtio_gpu_cmd_context_create()
930 	virtio_gpu_queue_ctrl_buffer(vgdev, vbuf);  in virtio_gpu_cmd_context_create()
937 	struct virtio_gpu_vbuffer *vbuf;  in virtio_gpu_cmd_context_destroy()  local
939 	cmd_p = virtio_gpu_alloc_cmd(vgdev, &vbuf, sizeof(*cmd_p));  in virtio_gpu_cmd_context_destroy()
944 	virtio_gpu_queue_ctrl_buffer(vgdev, vbuf);  in virtio_gpu_cmd_context_destroy()
953 	struct virtio_gpu_vbuffer *vbuf;  in virtio_gpu_cmd_context_attach_resource()  local
955 	cmd_p = virtio_gpu_alloc_cmd(vgdev, &vbuf, sizeof(*cmd_p));  in virtio_gpu_cmd_context_attach_resource()
957 	vbuf->objs = objs;  in virtio_gpu_cmd_context_attach_resource()
962 	virtio_gpu_queue_ctrl_buffer(vgdev, vbuf);  in virtio_gpu_cmd_context_attach_resource()
971 	struct virtio_gpu_vbuffer *vbuf;  in virtio_gpu_cmd_context_detach_resource()  local
973 	cmd_p = virtio_gpu_alloc_cmd(vgdev, &vbuf, sizeof(*cmd_p));  in virtio_gpu_cmd_context_detach_resource()
975 	vbuf->objs = objs;  in virtio_gpu_cmd_context_detach_resource()
980 	virtio_gpu_queue_ctrl_buffer(vgdev, vbuf);  in virtio_gpu_cmd_context_detach_resource()
991 	struct virtio_gpu_vbuffer *vbuf;  in virtio_gpu_cmd_resource_create_3d()  local
993 	cmd_p = virtio_gpu_alloc_cmd(vgdev, &vbuf, sizeof(*cmd_p));  in virtio_gpu_cmd_resource_create_3d()
995 	vbuf->objs = objs;  in virtio_gpu_cmd_resource_create_3d()
1011 	virtio_gpu_queue_fenced_ctrl_buffer(vgdev, vbuf, fence);  in virtio_gpu_cmd_resource_create_3d()
1025 	struct virtio_gpu_vbuffer *vbuf;  in virtio_gpu_cmd_transfer_to_host_3d()  local
1033 	cmd_p = virtio_gpu_alloc_cmd(vgdev, &vbuf, sizeof(*cmd_p));  in virtio_gpu_cmd_transfer_to_host_3d()
1036 	vbuf->objs = objs;  in virtio_gpu_cmd_transfer_to_host_3d()
1045 	virtio_gpu_queue_fenced_ctrl_buffer(vgdev, vbuf, fence);  in virtio_gpu_cmd_transfer_to_host_3d()
1057 	struct virtio_gpu_vbuffer *vbuf;  in virtio_gpu_cmd_transfer_from_host_3d()  local
1059 	cmd_p = virtio_gpu_alloc_cmd(vgdev, &vbuf, sizeof(*cmd_p));  in virtio_gpu_cmd_transfer_from_host_3d()
1062 	vbuf->objs = objs;  in virtio_gpu_cmd_transfer_from_host_3d()
1071 	virtio_gpu_queue_fenced_ctrl_buffer(vgdev, vbuf, fence);  in virtio_gpu_cmd_transfer_from_host_3d()
1081 	struct virtio_gpu_vbuffer *vbuf;  in virtio_gpu_cmd_submit()  local
1083 	cmd_p = virtio_gpu_alloc_cmd(vgdev, &vbuf, sizeof(*cmd_p));  in virtio_gpu_cmd_submit()
1086 	vbuf->data_buf = data;  in virtio_gpu_cmd_submit()
1087 	vbuf->data_size = data_size;  in virtio_gpu_cmd_submit()
1088 	vbuf->objs = objs;  in virtio_gpu_cmd_submit()
1094 	virtio_gpu_queue_fenced_ctrl_buffer(vgdev, vbuf, fence);  in virtio_gpu_cmd_submit()
1109 	struct virtio_gpu_vbuffer *vbuf;  in virtio_gpu_cursor_ping()  local
1113 	cur_p = virtio_gpu_alloc_cursor(vgdev, &vbuf);  in virtio_gpu_cursor_ping()
1115 	virtio_gpu_queue_cursor(vgdev, vbuf);  in virtio_gpu_cursor_ping()
1119 					    struct virtio_gpu_vbuffer *vbuf)  in virtio_gpu_cmd_resource_uuid_cb()  argument
1122 		gem_to_virtio_gpu_obj(vbuf->objs->objs[0]);  in virtio_gpu_cmd_resource_uuid_cb()
1124 		(struct virtio_gpu_resp_resource_uuid *)vbuf->resp_buf;  in virtio_gpu_cmd_resource_uuid_cb()
1148 	struct virtio_gpu_vbuffer *vbuf;  in virtio_gpu_cmd_resource_assign_uuid()  local
1161 		(vgdev, virtio_gpu_cmd_resource_uuid_cb, &vbuf, sizeof(*cmd_p),  in virtio_gpu_cmd_resource_assign_uuid()
1168 	vbuf->objs = objs;  in virtio_gpu_cmd_resource_assign_uuid()
1169 	virtio_gpu_queue_ctrl_buffer(vgdev, vbuf);  in virtio_gpu_cmd_resource_assign_uuid()