94 	struct virtio_gpu_vbuffer *vbuf;  in virtio_gpu_get_vbuf()  local
96 	vbuf = kmem_cache_zalloc(vgdev->vbufs, GFP_KERNEL | __GFP_NOFAIL);  in virtio_gpu_get_vbuf()
100 	vbuf->buf = (void *)vbuf + sizeof(*vbuf);  in virtio_gpu_get_vbuf()
101 	vbuf->size = size;  in virtio_gpu_get_vbuf()
103 	vbuf->resp_cb = resp_cb;  in virtio_gpu_get_vbuf()
104 	vbuf->resp_size = resp_size;  in virtio_gpu_get_vbuf()
106 		vbuf->resp_buf = (void *)vbuf->buf + size;  in virtio_gpu_get_vbuf()
108 		vbuf->resp_buf = resp_buf;  in virtio_gpu_get_vbuf()
109 	BUG_ON(!vbuf->resp_buf);  in virtio_gpu_get_vbuf()
110 	return vbuf;  in virtio_gpu_get_vbuf()
114 virtio_gpu_vbuf_ctrl_hdr(struct virtio_gpu_vbuffer *vbuf)  in virtio_gpu_vbuf_ctrl_hdr()  argument
120 	return (struct virtio_gpu_ctrl_hdr *)vbuf->buf;  in virtio_gpu_vbuf_ctrl_hdr()
127 	struct virtio_gpu_vbuffer *vbuf;  in virtio_gpu_alloc_cursor()  local
129 	vbuf = virtio_gpu_get_vbuf  in virtio_gpu_alloc_cursor()
132 	if (IS_ERR(vbuf)) {  in virtio_gpu_alloc_cursor()
134 		return ERR_CAST(vbuf);  in virtio_gpu_alloc_cursor()
136 	*vbuffer_p = vbuf;  in virtio_gpu_alloc_cursor()
137 	return (struct virtio_gpu_update_cursor *)vbuf->buf;  in virtio_gpu_alloc_cursor()
146 	struct virtio_gpu_vbuffer *vbuf;  in virtio_gpu_alloc_cmd_resp()  local
148 	vbuf = virtio_gpu_get_vbuf(vgdev, cmd_size,  in virtio_gpu_alloc_cmd_resp()
150 	*vbuffer_p = vbuf;  in virtio_gpu_alloc_cmd_resp()
151 	return (struct virtio_gpu_command *)vbuf->buf;  in virtio_gpu_alloc_cmd_resp()
174 		      struct virtio_gpu_vbuffer *vbuf)  in free_vbuf()  argument
176 	if (vbuf->resp_size > MAX_INLINE_RESP_SIZE)  in free_vbuf()
177 		kfree(vbuf->resp_buf);  in free_vbuf()
178 	kvfree(vbuf->data_buf);  in free_vbuf()
179 	kmem_cache_free(vgdev->vbufs, vbuf);  in free_vbuf()
184 	struct virtio_gpu_vbuffer *vbuf;  in reclaim_vbufs()  local
188 	while ((vbuf = virtqueue_get_buf(vq, &len))) {  in reclaim_vbufs()
189 		list_add_tail(&vbuf->list, reclaim_list);  in reclaim_vbufs()
311 				     struct virtio_gpu_vbuffer *vbuf,  in virtio_gpu_queue_ctrl_sgs()  argument
322 		if (fence && vbuf->objs)  in virtio_gpu_queue_ctrl_sgs()
323 			virtio_gpu_array_unlock_resv(vbuf->objs);  in virtio_gpu_queue_ctrl_sgs()
324 		free_vbuf(vgdev, vbuf);  in virtio_gpu_queue_ctrl_sgs()
345 		virtio_gpu_fence_emit(vgdev, virtio_gpu_vbuf_ctrl_hdr(vbuf),  in virtio_gpu_queue_ctrl_sgs()
347 		if (vbuf->objs) {  in virtio_gpu_queue_ctrl_sgs()
348 			virtio_gpu_array_add_fence(vbuf->objs, &fence->f);  in virtio_gpu_queue_ctrl_sgs()
349 			virtio_gpu_array_unlock_resv(vbuf->objs);  in virtio_gpu_queue_ctrl_sgs()
353 	ret = virtqueue_add_sgs(vq, sgs, outcnt, incnt, vbuf, GFP_ATOMIC);  in virtio_gpu_queue_ctrl_sgs()
356 	trace_virtio_gpu_cmd_queue(vq, virtio_gpu_vbuf_ctrl_hdr(vbuf));  in virtio_gpu_queue_ctrl_sgs()
367 					       struct virtio_gpu_vbuffer *vbuf,  in virtio_gpu_queue_fenced_ctrl_buffer()  argument
375 	sg_init_one(&vcmd, vbuf->buf, vbuf->size);  in virtio_gpu_queue_fenced_ctrl_buffer()
381 	if (vbuf->data_size) {  in virtio_gpu_queue_fenced_ctrl_buffer()
382 		if (is_vmalloc_addr(vbuf->data_buf)) {  in virtio_gpu_queue_fenced_ctrl_buffer()
384 			sgt = vmalloc_to_sgt(vbuf->data_buf, vbuf->data_size,  in virtio_gpu_queue_fenced_ctrl_buffer()
387 				if (fence && vbuf->objs)  in virtio_gpu_queue_fenced_ctrl_buffer()
388 					virtio_gpu_array_unlock_resv(vbuf->objs);  in virtio_gpu_queue_fenced_ctrl_buffer()
395 			sg_init_one(&vout, vbuf->data_buf, vbuf->data_size);  in virtio_gpu_queue_fenced_ctrl_buffer()
403 	if (vbuf->resp_size) {  in virtio_gpu_queue_fenced_ctrl_buffer()
404 		sg_init_one(&vresp, vbuf->resp_buf, vbuf->resp_size);  in virtio_gpu_queue_fenced_ctrl_buffer()
410 	ret = virtio_gpu_queue_ctrl_sgs(vgdev, vbuf, fence, elemcnt, sgs, outcnt,  in virtio_gpu_queue_fenced_ctrl_buffer()
437 					struct virtio_gpu_vbuffer *vbuf)  in virtio_gpu_queue_ctrl_buffer()  argument
439 	return virtio_gpu_queue_fenced_ctrl_buffer(vgdev, vbuf, NULL);  in virtio_gpu_queue_ctrl_buffer()
443 				    struct virtio_gpu_vbuffer *vbuf)  in virtio_gpu_queue_cursor()  argument
451 		free_vbuf(vgdev, vbuf);  in virtio_gpu_queue_cursor()
455 	sg_init_one(&ccmd, vbuf->buf, vbuf->size);  in virtio_gpu_queue_cursor()
461 	ret = virtqueue_add_sgs(vq, sgs, outcnt, 0, vbuf, GFP_ATOMIC);  in virtio_gpu_queue_cursor()
469 			virtio_gpu_vbuf_ctrl_hdr(vbuf));  in virtio_gpu_queue_cursor()
494 	struct virtio_gpu_vbuffer *vbuf;  in virtio_gpu_cmd_create_resource()  local
496 	cmd_p = virtio_gpu_alloc_cmd(vgdev, &vbuf, sizeof(*cmd_p));  in virtio_gpu_cmd_create_resource()
498 	vbuf->objs = objs;  in virtio_gpu_cmd_create_resource()
506 	virtio_gpu_queue_fenced_ctrl_buffer(vgdev, vbuf, fence);  in virtio_gpu_cmd_create_resource()
511 				    struct virtio_gpu_vbuffer *vbuf)  in virtio_gpu_cmd_unref_cb()  argument
515 	bo = vbuf->resp_cb_data;  in virtio_gpu_cmd_unref_cb()
516 	vbuf->resp_cb_data = NULL;  in virtio_gpu_cmd_unref_cb()
525 	struct virtio_gpu_vbuffer *vbuf;  in virtio_gpu_cmd_unref_resource()  local
528 	cmd_p = virtio_gpu_alloc_cmd_cb(vgdev, &vbuf, sizeof(*cmd_p),  in virtio_gpu_cmd_unref_resource()
535 	vbuf->resp_cb_data = bo;  in virtio_gpu_cmd_unref_resource()
536 	ret = virtio_gpu_queue_ctrl_buffer(vgdev, vbuf);  in virtio_gpu_cmd_unref_resource()
547 	struct virtio_gpu_vbuffer *vbuf;  in virtio_gpu_cmd_set_scanout()  local
549 	cmd_p = virtio_gpu_alloc_cmd(vgdev, &vbuf, sizeof(*cmd_p));  in virtio_gpu_cmd_set_scanout()
560 	virtio_gpu_queue_ctrl_buffer(vgdev, vbuf);  in virtio_gpu_cmd_set_scanout()
571 	struct virtio_gpu_vbuffer *vbuf;  in virtio_gpu_cmd_resource_flush()  local
573 	cmd_p = virtio_gpu_alloc_cmd(vgdev, &vbuf, sizeof(*cmd_p));  in virtio_gpu_cmd_resource_flush()
575 	vbuf->objs = objs;  in virtio_gpu_cmd_resource_flush()
584 	virtio_gpu_queue_fenced_ctrl_buffer(vgdev, vbuf, fence);  in virtio_gpu_cmd_resource_flush()
596 	struct virtio_gpu_vbuffer *vbuf;  in virtio_gpu_cmd_transfer_to_host_2d()  local
603 	cmd_p = virtio_gpu_alloc_cmd(vgdev, &vbuf, sizeof(*cmd_p));  in virtio_gpu_cmd_transfer_to_host_2d()
605 	vbuf->objs = objs;  in virtio_gpu_cmd_transfer_to_host_2d()
615 	virtio_gpu_queue_fenced_ctrl_buffer(vgdev, vbuf, fence);  in virtio_gpu_cmd_transfer_to_host_2d()
626 	struct virtio_gpu_vbuffer *vbuf;  in virtio_gpu_cmd_resource_attach_backing()  local
628 	cmd_p = virtio_gpu_alloc_cmd(vgdev, &vbuf, sizeof(*cmd_p));  in virtio_gpu_cmd_resource_attach_backing()
635 	vbuf->data_buf = ents;  in virtio_gpu_cmd_resource_attach_backing()
636 	vbuf->data_size = sizeof(*ents) * nents;  in virtio_gpu_cmd_resource_attach_backing()
638 	virtio_gpu_queue_fenced_ctrl_buffer(vgdev, vbuf, fence);  in virtio_gpu_cmd_resource_attach_backing()
642 					       struct virtio_gpu_vbuffer *vbuf)  in virtio_gpu_cmd_get_display_info_cb()  argument
645 		(struct virtio_gpu_resp_display_info *)vbuf->resp_buf;  in virtio_gpu_cmd_get_display_info_cb()
671 					      struct virtio_gpu_vbuffer *vbuf)  in virtio_gpu_cmd_get_capset_info_cb()  argument
674 		(struct virtio_gpu_get_capset_info *)vbuf->buf;  in virtio_gpu_cmd_get_capset_info_cb()
676 		(struct virtio_gpu_resp_capset_info *)vbuf->resp_buf;  in virtio_gpu_cmd_get_capset_info_cb()
692 				     struct virtio_gpu_vbuffer *vbuf)  in virtio_gpu_cmd_capset_cb()  argument
695 		(struct virtio_gpu_get_capset *)vbuf->buf;  in virtio_gpu_cmd_capset_cb()
697 		(struct virtio_gpu_resp_capset *)vbuf->resp_buf;  in virtio_gpu_cmd_capset_cb()
729 				       struct virtio_gpu_vbuffer *vbuf)  in virtio_gpu_cmd_get_edid_cb()  argument
732 		(struct virtio_gpu_cmd_get_edid *)vbuf->buf;  in virtio_gpu_cmd_get_edid_cb()
734 		(struct virtio_gpu_resp_edid *)vbuf->resp_buf;  in virtio_gpu_cmd_get_edid_cb()
758 	struct virtio_gpu_vbuffer *vbuf;  in virtio_gpu_cmd_get_display_info()  local
767 		(vgdev, &virtio_gpu_cmd_get_display_info_cb, &vbuf,  in virtio_gpu_cmd_get_display_info()
774 	virtio_gpu_queue_ctrl_buffer(vgdev, vbuf);  in virtio_gpu_cmd_get_display_info()
781 	struct virtio_gpu_vbuffer *vbuf;  in virtio_gpu_cmd_get_capset_info()  local
790 		(vgdev, &virtio_gpu_cmd_get_capset_info_cb, &vbuf,  in virtio_gpu_cmd_get_capset_info()
797 	virtio_gpu_queue_ctrl_buffer(vgdev, vbuf);  in virtio_gpu_cmd_get_capset_info()
806 	struct virtio_gpu_vbuffer *vbuf;  in virtio_gpu_cmd_get_capset()  local
865 		(vgdev, &virtio_gpu_cmd_capset_cb, &vbuf, sizeof(*cmd_p),  in virtio_gpu_cmd_get_capset()
872 	virtio_gpu_queue_ctrl_buffer(vgdev, vbuf);  in virtio_gpu_cmd_get_capset()
880 	struct virtio_gpu_vbuffer *vbuf;  in virtio_gpu_cmd_get_edids()  local
894 			(vgdev, &virtio_gpu_cmd_get_edid_cb, &vbuf,  in virtio_gpu_cmd_get_edids()
899 		virtio_gpu_queue_ctrl_buffer(vgdev, vbuf);  in virtio_gpu_cmd_get_edids()
910 	struct virtio_gpu_vbuffer *vbuf;  in virtio_gpu_cmd_context_create()  local
912 	cmd_p = virtio_gpu_alloc_cmd(vgdev, &vbuf, sizeof(*cmd_p));  in virtio_gpu_cmd_context_create()
921 	virtio_gpu_queue_ctrl_buffer(vgdev, vbuf);  in virtio_gpu_cmd_context_create()
928 	struct virtio_gpu_vbuffer *vbuf;  in virtio_gpu_cmd_context_destroy()  local
930 	cmd_p = virtio_gpu_alloc_cmd(vgdev, &vbuf, sizeof(*cmd_p));  in virtio_gpu_cmd_context_destroy()
935 	virtio_gpu_queue_ctrl_buffer(vgdev, vbuf);  in virtio_gpu_cmd_context_destroy()
944 	struct virtio_gpu_vbuffer *vbuf;  in virtio_gpu_cmd_context_attach_resource()  local
946 	cmd_p = virtio_gpu_alloc_cmd(vgdev, &vbuf, sizeof(*cmd_p));  in virtio_gpu_cmd_context_attach_resource()
948 	vbuf->objs = objs;  in virtio_gpu_cmd_context_attach_resource()
953 	virtio_gpu_queue_ctrl_buffer(vgdev, vbuf);  in virtio_gpu_cmd_context_attach_resource()
962 	struct virtio_gpu_vbuffer *vbuf;  in virtio_gpu_cmd_context_detach_resource()  local
964 	cmd_p = virtio_gpu_alloc_cmd(vgdev, &vbuf, sizeof(*cmd_p));  in virtio_gpu_cmd_context_detach_resource()
966 	vbuf->objs = objs;  in virtio_gpu_cmd_context_detach_resource()
971 	virtio_gpu_queue_ctrl_buffer(vgdev, vbuf);  in virtio_gpu_cmd_context_detach_resource()
982 	struct virtio_gpu_vbuffer *vbuf;  in virtio_gpu_cmd_resource_create_3d()  local
984 	cmd_p = virtio_gpu_alloc_cmd(vgdev, &vbuf, sizeof(*cmd_p));  in virtio_gpu_cmd_resource_create_3d()
986 	vbuf->objs = objs;  in virtio_gpu_cmd_resource_create_3d()
1002 	virtio_gpu_queue_fenced_ctrl_buffer(vgdev, vbuf, fence);  in virtio_gpu_cmd_resource_create_3d()
1018 	struct virtio_gpu_vbuffer *vbuf;  in virtio_gpu_cmd_transfer_to_host_3d()  local
1025 	cmd_p = virtio_gpu_alloc_cmd(vgdev, &vbuf, sizeof(*cmd_p));  in virtio_gpu_cmd_transfer_to_host_3d()
1028 	vbuf->objs = objs;  in virtio_gpu_cmd_transfer_to_host_3d()
1039 	virtio_gpu_queue_fenced_ctrl_buffer(vgdev, vbuf, fence);  in virtio_gpu_cmd_transfer_to_host_3d()
1053 	struct virtio_gpu_vbuffer *vbuf;  in virtio_gpu_cmd_transfer_from_host_3d()  local
1055 	cmd_p = virtio_gpu_alloc_cmd(vgdev, &vbuf, sizeof(*cmd_p));  in virtio_gpu_cmd_transfer_from_host_3d()
1058 	vbuf->objs = objs;  in virtio_gpu_cmd_transfer_from_host_3d()
1069 	virtio_gpu_queue_fenced_ctrl_buffer(vgdev, vbuf, fence);  in virtio_gpu_cmd_transfer_from_host_3d()
1079 	struct virtio_gpu_vbuffer *vbuf;  in virtio_gpu_cmd_submit()  local
1081 	cmd_p = virtio_gpu_alloc_cmd(vgdev, &vbuf, sizeof(*cmd_p));  in virtio_gpu_cmd_submit()
1084 	vbuf->data_buf = data;  in virtio_gpu_cmd_submit()
1085 	vbuf->data_size = data_size;  in virtio_gpu_cmd_submit()
1086 	vbuf->objs = objs;  in virtio_gpu_cmd_submit()
1092 	virtio_gpu_queue_fenced_ctrl_buffer(vgdev, vbuf, fence);  in virtio_gpu_cmd_submit()
1107 	struct virtio_gpu_vbuffer *vbuf;  in virtio_gpu_cursor_ping()  local
1111 	cur_p = virtio_gpu_alloc_cursor(vgdev, &vbuf);  in virtio_gpu_cursor_ping()
1113 	virtio_gpu_queue_cursor(vgdev, vbuf);  in virtio_gpu_cursor_ping()
1117 					    struct virtio_gpu_vbuffer *vbuf)  in virtio_gpu_cmd_resource_uuid_cb()  argument
1120 		gem_to_virtio_gpu_obj(vbuf->objs->objs[0]);  in virtio_gpu_cmd_resource_uuid_cb()
1122 		(struct virtio_gpu_resp_resource_uuid *)vbuf->resp_buf;  in virtio_gpu_cmd_resource_uuid_cb()
1146 	struct virtio_gpu_vbuffer *vbuf;  in virtio_gpu_cmd_resource_assign_uuid()  local
1159 		(vgdev, virtio_gpu_cmd_resource_uuid_cb, &vbuf, sizeof(*cmd_p),  in virtio_gpu_cmd_resource_assign_uuid()
1166 	vbuf->objs = objs;  in virtio_gpu_cmd_resource_assign_uuid()
1167 	virtio_gpu_queue_ctrl_buffer(vgdev, vbuf);  in virtio_gpu_cmd_resource_assign_uuid()
1172 					   struct virtio_gpu_vbuffer *vbuf)  in virtio_gpu_cmd_resource_map_cb()  argument
1175 		gem_to_virtio_gpu_obj(vbuf->objs->objs[0]);  in virtio_gpu_cmd_resource_map_cb()
1177 		(struct virtio_gpu_resp_map_info *)vbuf->resp_buf;  in virtio_gpu_cmd_resource_map_cb()
1199 	struct virtio_gpu_vbuffer *vbuf;  in virtio_gpu_cmd_map()  local
1207 		(vgdev, virtio_gpu_cmd_resource_map_cb, &vbuf, sizeof(*cmd_p),  in virtio_gpu_cmd_map()
1214 	vbuf->objs = objs;  in virtio_gpu_cmd_map()
1216 	virtio_gpu_queue_ctrl_buffer(vgdev, vbuf);  in virtio_gpu_cmd_map()
1224 	struct virtio_gpu_vbuffer *vbuf;  in virtio_gpu_cmd_unmap()  local
1226 	cmd_p = virtio_gpu_alloc_cmd(vgdev, &vbuf, sizeof(*cmd_p));  in virtio_gpu_cmd_unmap()
1232 	virtio_gpu_queue_ctrl_buffer(vgdev, vbuf);  in virtio_gpu_cmd_unmap()
1243 	struct virtio_gpu_vbuffer *vbuf;  in virtio_gpu_cmd_resource_create_blob()  local
1245 	cmd_p = virtio_gpu_alloc_cmd(vgdev, &vbuf, sizeof(*cmd_p));  in virtio_gpu_cmd_resource_create_blob()
1257 	vbuf->data_buf = ents;  in virtio_gpu_cmd_resource_create_blob()
1258 	vbuf->data_size = sizeof(*ents) * nents;  in virtio_gpu_cmd_resource_create_blob()
1260 	virtio_gpu_queue_ctrl_buffer(vgdev, vbuf);  in virtio_gpu_cmd_resource_create_blob()
1273 	struct virtio_gpu_vbuffer *vbuf;  in virtio_gpu_cmd_set_scanout_blob()  local
1276 	cmd_p = virtio_gpu_alloc_cmd(vgdev, &vbuf, sizeof(*cmd_p));  in virtio_gpu_cmd_set_scanout_blob()
1297 	virtio_gpu_queue_ctrl_buffer(vgdev, vbuf);  in virtio_gpu_cmd_set_scanout_blob()