30 	struct msm_gem_submit *submit;  in submit_create()  local
34 	sz = struct_size(submit, bos, nr_bos) +  in submit_create()
35 			((u64)nr_cmds * sizeof(submit->cmd[0]));  in submit_create()
40 	submit = kzalloc(sz, GFP_KERNEL);  in submit_create()
41 	if (!submit)  in submit_create()
44 	submit->hw_fence = msm_fence_alloc();  in submit_create()
45 	if (IS_ERR(submit->hw_fence)) {  in submit_create()
46 		ret = PTR_ERR(submit->hw_fence);  in submit_create()
47 		kfree(submit);  in submit_create()
51 	ret = drm_sched_job_init(&submit->base, queue->entity, queue);  in submit_create()
53 		kfree(submit->hw_fence);  in submit_create()
54 		kfree(submit);  in submit_create()
58 	kref_init(&submit->ref);  in submit_create()
59 	submit->dev = dev;  in submit_create()
60 	submit->aspace = queue->ctx->aspace;  in submit_create()
61 	submit->gpu = gpu;  in submit_create()
62 	submit->cmd = (void *)&submit->bos[nr_bos];  in submit_create()
63 	submit->queue = queue;  in submit_create()
64 	submit->pid = get_pid(task_pid(current));  in submit_create()
65 	submit->ring = gpu->rb[queue->ring_nr];  in submit_create()
66 	submit->fault_dumped = false;  in submit_create()
69 	submit->ident = atomic_inc_return(&ident) - 1;  in submit_create()
71 	INIT_LIST_HEAD(&submit->node);  in submit_create()
73 	return submit;  in submit_create()
78 	struct msm_gem_submit *submit =  in __msm_gem_submit_destroy()  local
82 	if (submit->fence_id) {  in __msm_gem_submit_destroy()
83 		spin_lock(&submit->queue->idr_lock);  in __msm_gem_submit_destroy()
84 		idr_remove(&submit->queue->fence_idr, submit->fence_id);  in __msm_gem_submit_destroy()
85 		spin_unlock(&submit->queue->idr_lock);  in __msm_gem_submit_destroy()
88 	dma_fence_put(submit->user_fence);  in __msm_gem_submit_destroy()
97 	if (kref_read(&submit->hw_fence->refcount) == 0) {  in __msm_gem_submit_destroy()
98 		kfree(submit->hw_fence);  in __msm_gem_submit_destroy()
100 		dma_fence_put(submit->hw_fence);  in __msm_gem_submit_destroy()
103 	put_pid(submit->pid);  in __msm_gem_submit_destroy()
104 	msm_submitqueue_put(submit->queue);  in __msm_gem_submit_destroy()
106 	for (i = 0; i < submit->nr_cmds; i++)  in __msm_gem_submit_destroy()
107 		kfree(submit->cmd[i].relocs);  in __msm_gem_submit_destroy()
109 	kfree(submit);  in __msm_gem_submit_destroy()
112 static int submit_lookup_objects(struct msm_gem_submit *submit,  in submit_lookup_objects()  argument
126 		submit->bos[i].flags = 0;  in submit_lookup_objects()
145 		submit->bos[i].handle = submit_bo.handle;  in submit_lookup_objects()
146 		submit->bos[i].flags = submit_bo.flags;  in submit_lookup_objects()
148 		submit->bos[i].iova  = submit_bo.presumed;  in submit_lookup_objects()
159 		obj = idr_find(&file->object_idr, submit->bos[i].handle);  in submit_lookup_objects()
161 			DRM_ERROR("invalid handle %u at index %u\n", submit->bos[i].handle, i);  in submit_lookup_objects()
168 		submit->bos[i].obj = obj;  in submit_lookup_objects()
175 	submit->nr_bos = i;  in submit_lookup_objects()
180 static int submit_lookup_cmds(struct msm_gem_submit *submit,  in submit_lookup_cmds()  argument
216 		submit->cmd[i].type = submit_cmd.type;  in submit_lookup_cmds()
217 		submit->cmd[i].size = submit_cmd.size / 4;  in submit_lookup_cmds()
218 		submit->cmd[i].offset = submit_cmd.submit_offset / 4;  in submit_lookup_cmds()
219 		submit->cmd[i].idx  = submit_cmd.submit_idx;  in submit_lookup_cmds()
220 		submit->cmd[i].nr_relocs = submit_cmd.nr_relocs;  in submit_lookup_cmds()
231 		submit->cmd[i].relocs = kmalloc(sz, GFP_KERNEL);  in submit_lookup_cmds()
232 		if (!submit->cmd[i].relocs) {  in submit_lookup_cmds()
236 		ret = copy_from_user(submit->cmd[i].relocs, userptr, sz);  in submit_lookup_cmds()
251 static void submit_cleanup_bo(struct msm_gem_submit *submit, int i,  in submit_cleanup_bo()  argument
254 	struct drm_gem_object *obj = submit->bos[i].obj;  in submit_cleanup_bo()
255 	unsigned flags = submit->bos[i].flags & cleanup_flags;  in submit_cleanup_bo()
262 	submit->bos[i].flags &= ~cleanup_flags;  in submit_cleanup_bo()
271 static void submit_unlock_unpin_bo(struct msm_gem_submit *submit, int i)  in submit_unlock_unpin_bo()  argument
274 	submit_cleanup_bo(submit, i, cleanup_flags);  in submit_unlock_unpin_bo()
276 	if (!(submit->bos[i].flags & BO_VALID))  in submit_unlock_unpin_bo()
277 		submit->bos[i].iova = 0;  in submit_unlock_unpin_bo()
281 static int submit_lock_objects(struct msm_gem_submit *submit)  in submit_lock_objects()  argument
286 	for (i = 0; i < submit->nr_bos; i++) {  in submit_lock_objects()
287 		struct drm_gem_object *obj = submit->bos[i].obj;  in submit_lock_objects()
294 		if (!(submit->bos[i].flags & BO_LOCKED)) {  in submit_lock_objects()
296 							  &submit->ticket);  in submit_lock_objects()
299 			submit->bos[i].flags |= BO_LOCKED;  in submit_lock_objects()
303 	ww_acquire_done(&submit->ticket);  in submit_lock_objects()
310 				submit->bos[i].handle, i);  in submit_lock_objects()
315 		submit_unlock_unpin_bo(submit, i);  in submit_lock_objects()
318 		submit_unlock_unpin_bo(submit, slow_locked);  in submit_lock_objects()
321 		struct drm_gem_object *obj = submit->bos[contended].obj;  in submit_lock_objects()
324 						       &submit->ticket);  in submit_lock_objects()
326 			submit->bos[contended].flags |= BO_LOCKED;  in submit_lock_objects()
341 static int submit_fence_sync(struct msm_gem_submit *submit, bool no_implicit)  in submit_fence_sync()  argument
345 	for (i = 0; i < submit->nr_bos; i++) {  in submit_fence_sync()
346 		struct drm_gem_object *obj = submit->bos[i].obj;  in submit_fence_sync()
347 		bool write = submit->bos[i].flags & MSM_SUBMIT_BO_WRITE;  in submit_fence_sync()
369 		if (submit->bos[i].flags & MSM_SUBMIT_BO_NO_IMPLICIT)  in submit_fence_sync()
372 		ret = drm_sched_job_add_implicit_dependencies(&submit->base,  in submit_fence_sync()
382 static int submit_pin_objects(struct msm_gem_submit *submit)  in submit_pin_objects()  argument
384 	struct msm_drm_private *priv = submit->dev->dev_private;  in submit_pin_objects()
387 	submit->valid = true;  in submit_pin_objects()
389 	for (i = 0; i < submit->nr_bos; i++) {  in submit_pin_objects()
390 		struct drm_gem_object *obj = submit->bos[i].obj;  in submit_pin_objects()
394 		vma = msm_gem_get_vma_locked(obj, submit->aspace);  in submit_pin_objects()
404 		if (vma->iova == submit->bos[i].iova) {  in submit_pin_objects()
405 			submit->bos[i].flags |= BO_VALID;  in submit_pin_objects()
407 			submit->bos[i].iova = vma->iova;  in submit_pin_objects()
409 			submit->bos[i].flags &= ~BO_VALID;  in submit_pin_objects()
410 			submit->valid = false;  in submit_pin_objects()
422 	for (i = 0; i < submit->nr_bos; i++) {  in submit_pin_objects()
423 		msm_gem_pin_obj_locked(submit->bos[i].obj);  in submit_pin_objects()
424 		submit->bos[i].flags |= BO_PINNED;  in submit_pin_objects()
431 static void submit_attach_object_fences(struct msm_gem_submit *submit)  in submit_attach_object_fences()  argument
435 	for (i = 0; i < submit->nr_bos; i++) {  in submit_attach_object_fences()
436 		struct drm_gem_object *obj = submit->bos[i].obj;  in submit_attach_object_fences()
438 		if (submit->bos[i].flags & MSM_SUBMIT_BO_WRITE)  in submit_attach_object_fences()
439 			dma_resv_add_fence(obj->resv, submit->user_fence,  in submit_attach_object_fences()
441 		else if (submit->bos[i].flags & MSM_SUBMIT_BO_READ)  in submit_attach_object_fences()
442 			dma_resv_add_fence(obj->resv, submit->user_fence,  in submit_attach_object_fences()
447 static int submit_bo(struct msm_gem_submit *submit, uint32_t idx,  in submit_bo()  argument
450 	if (idx >= submit->nr_bos) {  in submit_bo()
452 				idx, submit->nr_bos);  in submit_bo()
457 		*obj = submit->bos[idx].obj;  in submit_bo()
459 		*iova = submit->bos[idx].iova;  in submit_bo()
461 		*valid = !!(submit->bos[idx].flags & BO_VALID);  in submit_bo()
467 static int submit_reloc(struct msm_gem_submit *submit, struct drm_gem_object *obj,  in submit_reloc()  argument
516 		ret = submit_bo(submit, submit_reloc.reloc_idx, NULL, &iova, &valid);  in submit_reloc()
545 static void submit_cleanup(struct msm_gem_submit *submit, bool error)  in submit_cleanup()  argument
553 	for (i = 0; i < submit->nr_bos; i++) {  in submit_cleanup()
554 		struct drm_gem_object *obj = submit->bos[i].obj;  in submit_cleanup()
555 		submit_cleanup_bo(submit, i, cleanup_flags);  in submit_cleanup()
561 void msm_submit_retire(struct msm_gem_submit *submit)  in msm_submit_retire()  argument
565 	for (i = 0; i < submit->nr_bos; i++) {  in msm_submit_retire()
566 		struct drm_gem_object *obj = submit->bos[i].obj;  in msm_submit_retire()
578 static struct drm_syncobj **msm_parse_deps(struct msm_gem_submit *submit,  in msm_parse_deps()  argument
605 		    !drm_core_check_feature(submit->dev, DRIVER_SYNCOBJ_TIMELINE)) {  in msm_parse_deps()
615 		ret = drm_sched_job_add_syncobj_dependency(&submit->base, file,  in msm_parse_deps()
745 	struct msm_gem_submit *submit = NULL;  in msm_ioctl_gem_submit()  local
796 	submit = submit_create(dev, gpu, queue, args->nr_bos, args->nr_cmds);  in msm_ioctl_gem_submit()
797 	if (IS_ERR(submit)) {  in msm_ioctl_gem_submit()
798 		ret = PTR_ERR(submit);  in msm_ioctl_gem_submit()
802 	trace_msm_gpu_submit(pid_nr(submit->pid), ring->id, submit->ident,  in msm_ioctl_gem_submit()
810 		submit->in_rb = true;  in msm_ioctl_gem_submit()
822 		ret = drm_sched_job_add_dependency(&submit->base, in_fence);  in msm_ioctl_gem_submit()
828 		syncobjs_to_reset = msm_parse_deps(submit, file,  in msm_ioctl_gem_submit()
849 	ret = submit_lookup_objects(submit, args, file);  in msm_ioctl_gem_submit()
853 	ret = submit_lookup_cmds(submit, args, file);  in msm_ioctl_gem_submit()
858 	ww_acquire_init(&submit->ticket, &reservation_ww_class);  in msm_ioctl_gem_submit()
860 	ret = submit_lock_objects(submit);  in msm_ioctl_gem_submit()
864 	ret = submit_fence_sync(submit, !!(args->flags & MSM_SUBMIT_NO_IMPLICIT));  in msm_ioctl_gem_submit()
868 	ret = submit_pin_objects(submit);  in msm_ioctl_gem_submit()
876 		ret = submit_bo(submit, submit->cmd[i].idx,  in msm_ioctl_gem_submit()
881 		if (!submit->cmd[i].size ||  in msm_ioctl_gem_submit()
882 			((submit->cmd[i].size + submit->cmd[i].offset) >  in msm_ioctl_gem_submit()
884 			DRM_ERROR("invalid cmdstream size: %u\n", submit->cmd[i].size * 4);  in msm_ioctl_gem_submit()
889 		submit->cmd[i].iova = iova + (submit->cmd[i].offset * 4);  in msm_ioctl_gem_submit()
891 		if (submit->valid)  in msm_ioctl_gem_submit()
895 			if (submit->cmd[i].nr_relocs) {  in msm_ioctl_gem_submit()
904 		ret = submit_reloc(submit, obj, submit->cmd[i].offset * 4,  in msm_ioctl_gem_submit()
905 				submit->cmd[i].nr_relocs, submit->cmd[i].relocs);  in msm_ioctl_gem_submit()
910 	submit->nr_cmds = i;  in msm_ioctl_gem_submit()
930 	drm_sched_job_arm(&submit->base);  in msm_ioctl_gem_submit()
932 	submit->user_fence = dma_fence_get(&submit->base.s_fence->finished);  in msm_ioctl_gem_submit()
940 		submit->fence_id = args->fence;  in msm_ioctl_gem_submit()
941 		ret = idr_alloc_u32(&queue->fence_idr, submit->user_fence,  in msm_ioctl_gem_submit()
942 				    &submit->fence_id, submit->fence_id,  in msm_ioctl_gem_submit()
954 		submit->fence_id = idr_alloc_cyclic(&queue->fence_idr,  in msm_ioctl_gem_submit()
955 						    submit->user_fence, 1,  in msm_ioctl_gem_submit()
962 	if (submit->fence_id < 0) {  in msm_ioctl_gem_submit()
963 		ret = submit->fence_id;  in msm_ioctl_gem_submit()
964 		submit->fence_id = 0;  in msm_ioctl_gem_submit()
968 		struct sync_file *sync_file = sync_file_create(submit->user_fence);  in msm_ioctl_gem_submit()
977 	submit_attach_object_fences(submit);  in msm_ioctl_gem_submit()
980 	msm_gem_submit_get(submit);  in msm_ioctl_gem_submit()
982 	msm_rd_dump_submit(priv->rd, submit, NULL);  in msm_ioctl_gem_submit()
984 	drm_sched_entity_push_job(&submit->base);  in msm_ioctl_gem_submit()
986 	args->fence = submit->fence_id;  in msm_ioctl_gem_submit()
987 	queue->last_fence = submit->fence_id;  in msm_ioctl_gem_submit()
991 	                      submit->user_fence);  in msm_ioctl_gem_submit()
995 	submit_cleanup(submit, !!ret);  in msm_ioctl_gem_submit()
997 		ww_acquire_fini(&submit->ticket);  in msm_ioctl_gem_submit()
1004 	if (!IS_ERR_OR_NULL(submit)) {  in msm_ioctl_gem_submit()
1005 		msm_gem_submit_put(submit);  in msm_ioctl_gem_submit()